# robots.txt
# Last updated: 2026-03-12

#  AI / LLM training crawlers 

User-agent: GPTBot
Crawl-delay: 3600

User-agent: ChatGPT-User
Crawl-delay: 3600

User-agent: OAI-SearchBot
Crawl-delay: 3600

User-agent: anthropic-ai
Crawl-delay: 3600

User-agent: ClaudeBot
Crawl-delay: 3600

User-agent: Claude-Web
Crawl-delay: 3600

User-agent: cohere-ai
Crawl-delay: 3600

User-agent: PerplexityBot
Crawl-delay: 3600

User-agent: YouBot
Crawl-delay: 3600

User-agent: Omgilibot
Crawl-delay: 3600

User-agent: FacebookBot
Crawl-delay: 3600

User-agent: Meta-ExternalAgent
Crawl-delay: 3600

User-agent: Meta-ExternalFetcher
Crawl-delay: 3600

User-agent: Applebot-Extended
Crawl-delay: 3600

User-agent: Bytespider
Crawl-delay: 3600

User-agent: CCBot
Crawl-delay: 3600

User-agent: DataForSeoBot
Crawl-delay: 3600

User-agent: ImagesiftBot
Crawl-delay: 3600

User-agent: magpie-crawler
Crawl-delay: 3600

User-agent: Diffbot
Crawl-delay: 3600

User-agent: Timpibot
Crawl-delay: 3600

User-agent: PetalBot
Crawl-delay: 3600

User-agent: Scrapy
Crawl-delay: 3600

User-agent: SemrushBot-OCOB
Crawl-delay: 3600

#  Standard search engine crawlers (normal cadence) 

User-agent: Googlebot
Crawl-delay: 10

User-agent: Bingbot
Crawl-delay: 10


#  All other bots 

User-agent: *
Crawl-delay: 30

# Sitemap (update to your actual URL)
Sitemap: https://www.tatonkashop.cz/sitemap.xml