# robots.txt made with The Crawler Zoo
# https://crawlerzoo.com/robots-txt-generator?b=adsbot-google,google-display-ads-bot,googleother,mediapartners-google,storebot-google,agenttrustbot,ai2bot,amazonbot,applebot-extended,bytespider,ccbot,chatgpt-user,claude-searchbot,claude-user,claudebot,cohere,diffbot,duckassistbot,google-extended,gptbot,imagesiftbot,meta-externalagent,mistralai-user,oai-searchbot,perplexity-user,perplexitybot,reflectionbot,youbot,ahrefsbot,awariobot,barkrowler,blexbot,dataforseobot,domainarrivalsbot,dotbot,megaindex,mj12bot,pandalytics,screaming-frog,semrushbot,serpstatbot,chamberbot,fossickbot,scrapy,srchs-research-bot,wget
#
# Blocked: AdsBot-Google (Google), Google-Display-Ads-Bot (Google), GoogleOther (Google), Mediapartners-Google (Google), Storebot-Google (Google), AgentTrustBot (AgentTrust), AI2Bot (Allen Institute for AI), Amazonbot (Amazon), Applebot-Extended (Apple), Bytespider (ByteDance), CCBot (Common Crawl), ChatGPT-User (OpenAI), Claude-SearchBot (Anthropic), Claude-User (Anthropic), ClaudeBot (Anthropic), cohere-ai (Cohere), Diffbot (Diffbot), DuckAssistBot (DuckDuckGo), Google-Extended (Google), GPTBot (OpenAI), ImagesiftBot (Hive), Meta-ExternalAgent (Meta), MistralAI-User (Mistral AI), OAI-SearchBot (OpenAI), Perplexity-User (Perplexity), PerplexityBot (Perplexity), Reflectionbot (Reflection AI), YouBot (You.com), AhrefsBot (Ahrefs), AwarioBot (Awario), Barkrowler (Babbar), BLEXBot (WebMeUp), DataForSeoBot (DataForSEO), DomainArrivalsBot (DomainArrivals), DotBot (Moz), MegaIndex (MegaIndex), MJ12bot (Majestic), Pandalytics (DomainsBot), Screaming Frog SEO Spider (Screaming Frog), SemrushBot (Semrush), SerpstatBot (Serpstat), ChamberBot (Unknown), FossickBot (fossick.bot), Scrapy crawler (Unknown), srchs-research-bot (An individual researcher), Wget command (Unknown)

User-agent: AdsBot-Google
User-agent: Google-Display-Ads-Bot
User-agent: GoogleOther
User-agent: Mediapartners-Google
User-agent: Storebot-Google
User-agent: AgentTrustBot
User-agent: AI2Bot
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: ClaudeBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: DuckAssistBot
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ImagesiftBot
User-agent: meta-externalagent
User-agent: MistralAI-User
User-agent: OAI-SearchBot
User-agent: Perplexity-User
User-agent: PerplexityBot
User-agent: Reflectionbot
User-agent: YouBot
User-agent: AhrefsBot
User-agent: AwarioBot
User-agent: Barkrowler
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: DomainArrivalsBot
User-agent: DotBot
User-agent: MegaIndex.ru
User-agent: MJ12bot
User-agent: Pandalytics
User-agent: Screaming Frog SEO Spider
User-agent: SemrushBot
User-agent: serpstatbot
User-agent: ChamberBot
User-agent: FossickBot
User-agent: Scrapy
User-agent: srchs-research-bot
User-agent: Wget
Disallow: /

# Everyone else may read the whole site.
User-agent: *
Allow: /


Sitemap: https://tk-web.quest/sitemap.xml