User-Agent: *
Allow: /
Disallow: /auth/
Disallow: /dashboard
Disallow: /developers/keys
Disallow: /developers/signup
Disallow: /login
Crawl-delay: 2

User-Agent: GPTBot
Allow: /

User-Agent: ClaudeBot
Allow: /

User-Agent: PerplexityBot
Allow: /

User-Agent: Google-Extended
Allow: /

User-Agent: CCBot
Allow: /

User-Agent: AhrefsBot
Disallow: /api/

User-Agent: SemrushBot
Disallow: /api/

User-Agent: MJ12bot
Disallow: /api/

User-Agent: DotBot
Disallow: /api/

Host: https://worldoftaxonomy.com
Sitemap: https://worldoftaxonomy.com/sitemap.xml