Feature bots yaml

.gitignore

@@ -1,3 +1,4 @@
 .Build
 .php_cs.cache
 Documentation-GENERATED-temp
+regex.txt

Classes/Middleware/RedirectionMiddleware.php

@@ -26,7 +26,7 @@ class RedirectionMiddleware implements MiddlewareInterface
     /**
      * @var string
      */
-    protected $botPattern = '/bot|google|baidu|bing|msn|teoma|slurp|yandex|Chrome-Lighthouse/i';
+	protected $botPattern = '#(360Spider)|(Aboundex)|(AcoonBot)|(AddThis\.com)|(AhrefsBot)|(ia_archiver|alexabot|verifybot)|(alexa site audit)|(Amazonbot)|(Amazon[ -]Route ?53[ -]Health[ -]Check[ -]Service)|(AmorankSpider)|(ApacheBench)|(Applebot)|(AppSignalBot)|(Arachni)|(AspiegelBot)|(Castro 2, Episode Duration Lookup)|(Curious George)|(archive\.org_bot|special_archiver)|(Ask Jeeves/Teoma)|(Backlink-Check\.de)|(BacklinkCrawler)|(Baidu.*spider|baidu Transcoder)|(BazQux)|(Better Uptime Bot)|(MSNBot|msrbot|bingbot|BingPreview|msnbot-(UDiscovery|NewsBlogs)|adidxbot)|(Blekkobot)|(BLEXBot)|(Bloglovin)|(Blogtrottr)|(BoardReader Blog Indexer)|(BountiiBot)|(Browsershots)|(BUbiNG)|((?<!HTC)[ _]Butterfly/)|(CareerBot)|(CCBot)|(Cliqzbot)|(Cloudflare-AMP)|(CloudflareDiagnostics)|(CloudFlare-AlwaysOnline)|(coccoc.com)|(collectd)|(CommaFeed)|(CSS Certificate Spider)|(Datadog Agent)|(Datanyze)|(Dataprovider)|(Daum(oa)?[ /][0-9])|(Dazoobot)|(discobot)|(Domain Re-Animator Bot|[email protected])|(DotBot)|(DuckDuck(?:Go-Favicons-)?Bot)|(EasouSpider)|(eCairn-Grabber)|(EMail Exractor)|(evc-batch)|(Exabot|ExaleadCloudview)|(ExactSeek Crawler)|(Ezooms)|(facebookexternalhit|facebookplatform|facebookexternalua|facebookcatalog)|(Feedbin)|(FeedBurner)|(Feed Wrangler)|(Feedly)|(Feedspot)|(Fever/[0-9])|(FlipboardProxy|FlipboardRSS)|(Findxbot)|(FreshRSS)|(Genieo)|(GigablastOpenSource)|(Gluten Free Crawler)|(gobuster)|(ichiro/mobile goo)|(Storebot-Google)|(Google Favicon)|(Google Search Console)|(Google Page Speed Insights)|(google_partner_monitoring)|(Google-Cloud-Scheduler)|(Google-Structured-Data-Testing-Tool)|(GoogleStackdriverMonitoring)|(via ggpht\.com GoogleImageProxy)|(SeznamEmailProxy)|(Seznam-Zbozi-robot)|(Heurekabot-Feed)|(ShopAlike)|(AdsBot-Google|Adwords-(DisplayAds|Express|Instant)|Google Web Preview|Google[ -]Publisher[ -]Plugin|Google-(Ads-Qualify|Adwords|AMPHTML|Assess|HotelAdsVerifier|Read-Aloud|Shopping-Quality|Site-Verification|speakr|Test|Youtube-Links)|(APIs|DuplexWeb|Feedfetcher|Mediapartners)-Google|Googlebot|GoogleProducer|Google.*/\+/web/snippet)|(heritrix)|(HubSpot )|(HTTPMon)|(ICC-Crawler)|(inoreader.com)|(iisbot)|(ips-agent)|(IP-Guide\.com)|(k6/[0-9\.]+)|(kouio)|(larbin)|(([A-z0-9]*)-Lighthouse)|(linkdexbot|linkdex\.com)|(LinkedInBot)|(ltx71)|(Mail\.RU)|(magpie-crawler)|(MagpieRSS)|(masscan)|(Mastodon/)|(meanpathbot)|(MetaJobBot)|(MetaInspector)|(MixrankBot)|(MJ12bot)|(Mnogosearch)|(MojeekBot)|(munin)|(NalezenCzBot)|(check_http/v)|(nbertaupete95\(at\)gmail.com)|(Netcraft( Web Server Survey| SSL Server Survey|SurveyAgent))|(netEstate NE Crawler)|(Netvibes)|(NewsBlur .*(Fetcher|Finder))|(NewsGatorOnline)|(nlcrawler)|(Nmap Scripting Engine)|(Nuzzel)|(Octopus [0-9])|(omgili)|(OpenindexSpider)|(spbot)|(OpenWebSpider)|(OrangeBot|VoilaBot)|(PaperLiBot)|(phantomas/)|(phpservermon)|(PocketParser)|(PritTorrent)|(PRTG Network Monitor)|(psbot)|(Pingdom(?:\.com|TMS))|(Quora Link Preview)|(Quora-Bot)|(RamblerMail)|(QuerySeekerSpider)|(Qwantify)|(Rainmeter)|(redditbot)|(Riddler)|(rogerbot)|(ROI Hunter)|(SafeDNSBot)|(Scrapy)|(Screaming Frog SEO Spider)|(ScreenerBot)|(SemrushBot)|(SensikaBot)|(SEOENG(World)?Bot)|(SEOkicks-Robot)|(seoscanners\.net)|(SkypeUriPreview)|(SeznamBot|SklikBot|Seznam screenshot-generator)|(shopify-partner-homepage-scraper)|(ShopWiki)|(SilverReader)|(SimplePie)|(SISTRIX Crawler)|(compatible; (?:SISTRIX )?Optimizer)|(SiteSucker)|(sixy.ch)|(Slackbot|Slack-ImgProxy)|((Sogou (web|inst|Pic) spider)|New-Sogou-Spider)|(Sosospider|Sosoimagespider)|(Sprinklr)|(sqlmap/)|(SSL Labs)|(StatusCake)|(Superfeedr bot)|(Sparkler/[0-9])|(Spinn3r)|(SputnikBot)|(SputnikFaviconBot)|(SputnikImageBot)|(SurveyBot)|(TarmotGezgin)|(TelegramBot)|(TLSProbe)|(TinEye-bot)|(Tiny Tiny RSS)|(theoldreader.com)|(trendictionbot)|(TurnitinBot)|(TweetedTimes Bot)|(TweetmemeBot)|(Twingly Recon)|(Twitterbot)|(UniversalFeedParser)|(via secureurl\.fwdcdn\.com)|(Uptimebot)|(UptimeRobot)|(URLAppendBot)|(Vagabondo)|(vkShare; )|(VSMCrawler)|(Jigsaw)|(W3C_I18n-Checker)|(W3C-checklink)|(W3C_Validator|Validator.nu)|(W3C-mobileOK)|(W3C_Unicorn)|(Wappalyzer)|(PTST/)|(WeSEE)|(WebbCrawler)|(websitepulse[+ ]checker)|(WordPress)|(Wotbox)|(XenForo)|(yacybot)|(Yahoo! Slurp|Yahoo!-AdCrawler)|(Yahoo Link Preview|Yahoo:LinkExpander:Slingstone)|(YahooMailProxy)|(YahooCacheSystem)|(Y!J-BRW)|(Yandex(SpravBot|ScreenshotBot|MobileBot|AccessibilityBot|ForDomain|Vertis|Market|Catalog|Calendar|Sitelinks|AdNet|Pagechecker|Webmaster|Media|Video|Bot|Images|Antivirus|Direct|Blogs|Favicons|ImageResizer|Verticals|News|Metrika|\.Gazeta Bot)|YaDirectFetcher|YandexTurbo|YandexTracker|YandexSearchShop|YandexRCA|YandexPartner|YandexOntoDBAPI|YandexOntoDB|YandexMobileScreenShotBot)|(Yeti|NaverJapan)|(YoudaoBot)|(YOURLS v[0-9])|(YRSpider|YYSpider)|(zgrab)|(Zookabot)|(ZumBot)|(YottaaMonitor)|(Yahoo Ad monitoring.*yahoo-ad-monitoring-SLN24857.*)|(.*Java.*outbrain)|(HubPages.*crawlingpolicy)|(Pinterest(bot)?/\d\.\d.*www\.pinterest\.com.*)|(Site24x7)|(s~snapchat-proxy)|(Let\'s Encrypt validation server)|(GrapeshotCrawler)|(www\.monitor\.us)|(Catchpoint)|(bitlybot)|(Zao/)|(lycos)|(Slurp)|(Speedy Spider)|(ScoutJet)|(nrsbot|netresearch)|(scooter)|(gigabot)|(charlotte)|(Pompos)|(ichiro)|(PagePeeker)|(WebThumbnail)|(Willow Internet Crawler)|(EmailWolf)|(NetLyzer FastProbe)|(AdMantX.*admantx\.com)|(Server Density Service Monitoring.*)|(RSSRadio \(Push Notification Scanner;support@dorada\.co\.uk\))|((A6-Indexer|nuhk|TsolCrawler|Yammybot|Openbot|Gulper Web Bot|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr.com|tweetedtimes.com|TrendsmapResolver|teoma|blitzbot|oegp|furlbot|http%20client|polybot|htdig|mogimogi|larbin|scrubby|searchsight|seekbot|semanticdiscovery|snappy|vortex(?! Build)|zeal|fast-webcrawler|converacrawler|dataparksearch|findlinks|BrowserMob|HttpMonitor|ThumbShotsBot|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|RackspaceBot|robots|SeopultContentAnalyzer|7Siters|centuryb.o.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|My User Agent))|(^sentry)|(^Spotify)|(The Knowledge AI)|(Embedly)|(BrandVerity)|(Kaspersky Lab CFR link resolver)|(eZ Publish Link Validator)|(woorankreview)|((Match|LinkCheck) by Siteimprove.com)|(CATExplorador)|(Buck)|(tracemyfile)|(zelist.ro feed parser)|(weborama-fetcher)|(BoardReader Favicon Fetcher)|(IDG/IT)|(Bytespider)|(WikiDo)|(AwarioSmartBot)|(AwarioRssBot)|(oBot)|(SMTBot)|(LCC)|(Startpagina-Linkchecker)|(GTmetrix)|(Nutch)|(Seobility)|(Vercelbot)|(Grammarly)|(Robozilla)|(Domains Project)|(PetalBot)|(SerendeputyBot)|(ias-va.*admantx.*service-fetcher)|(SemanticScholarBot)|(VelenPublicWebCrawler)|(Barkrowler)|(BDCbot)|(adbeat)|(BW/(?:(\d+[\.\d]+)))|(https://whatis.contentkingapp.com)|(MicroAdBot)|(PingAdmin.Ru)|(notifyninja.+monitoring)|(WebDataStats)|(parse.ly scraper)|(Nimbostratus-Bot)|(HeartRails_Capture/\d)|(Project-Resonance)|(DataXu/\d)|(Cocolyzebot)|(veryhip)|(LinkpadBot)|(MuscatFerret)|(PageThing.com)|(ArchiveBox)|(Choosito)|(datagnionbot)|(WhatCMS)|(httpx)|([email protected])|(HuaweiWebCatBot)|(Hatena-Favicon)|(RyowlEngine/(\d+))|(OdklBot/(\d+))|(Mediatoolkitbot)|(ZoominfoBot)|(WeViKaBot/([\d+\.]))|(SEOkicks)|(Plukkie/([\d+\.]))|(proximic;)|(SurdotlyBot/([\d+\.]))|(Gowikibot/([\d+\.]))|(SabsimBot/([\d+\.]))|(LumtelBot/([\d+\.]))|(PiplBot)|(woobot/([\d+\.]))|(Cookiebot/([\d+\.]))|(NetSystemsResearch)|(CensysInspect/([\d+\.]))|(gdnplus.com)|(WellKnownBot/([\d+\.]))|(Adsbot/([\d+\.]))|(MTRobot/([\d+\.]))|(serpstatbot/([\d+\.]))|(colly)|(l9tcpid/v([\d+\.]))|(MegaIndex.ru/([\d+\.]))|(Seekport)|(seolyt/([\d+\.]))|(YaK/([\d+\.]))|(KomodiaBot/([\d+\.]))|(Neevabot/([\d+\.]))|(LinkPreview/([\d+\.]))|(JungleKeyThumbnail/([\d+\.]))|(rocketmonitor(?: |bot/)([\d+\.]))|(SitemapParser-VIPnytt/([\d+\.]))|(^Turnitin)|(DMBrowser/\d+|DMBrowser-[UB]V)|(ThinkChaos/)|(DataForSeoBot)|(Discordbot/([\d+.]+))|([a-z0-9\-_]*((?<!cu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9])|crawler|crawl|checker|archiver|transcoder|spider)([^a-z]|$))#i';
 
     /**
      * Adds an instance of TYPO3\CMS\Core\Http\NormalizedParams as