Sitemap: https://marketingdrs.com/sitemap_index.xml

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /eff-ewe
Disallow: /login
Allow: /wp-admin/admin-ajax.php

# AI Policy Files
Allow: /ai.txt
Allow: /llms.txt

# Robots.txt file for AI
# Allow OpenAI GPTBot (for model training)
User-agent: GPTBot
Allow: /

# Allow OpenAI OAI-SearchBot (for SearchGPT)
User-agent: OAI-SearchBot
Allow: /

# Allow PerplexityBot (for Perplexity.ai search assistant)
User-agent: PerplexityBot
Allow: /

# Allow youBot (You.com AI Assistant)
User-agent: youBot
Allow: /

# Allow Googlebot (may be used by Bard/Gemini)
User-agent: Googlebot
Allow: /

# Allow Amazonbot (potential AI applications)
User-agent: Amazonbot
Allow: /

# Allow CCBot (used for Common Crawl, many AI model trainings)
User-agent: CCBot
Allow: /

# Allow Bytespider (ByteDance / TikTok – potential AI training)
User-agent: Bytespider
Allow: /

# Allow ClaudeBot (Anthropic's bot - not official, but emerging)
User-agent: ClaudeBot
Allow: /

# Allow FacebookExternalHit (used by Meta, could support LLaMA data)
User-agent: facebookexternalhit
Allow: /

# Allow Applebot (Siri, Apple Search, potential AI integrations)
User-agent: Applebot
Allow: /