User-agent: *
Allow: /
Disallow: /api/
Disallow: /internal/
Disallow: /admin/
Disallow: /private/
Disallow: /training/
Disallow: /*.json$
Disallow: /temp/
Disallow: /cache/

# Prevent indexing of response pages
Disallow: /response/
Disallow: /completion/
Disallow: /generate/

# Rate limiting for specific bots
User-agent: GPTBot
Crawl-delay: 5
Disallow: /api/

User-agent: ChatGPT-User
Crawl-delay: 5
Disallow: /api/

User-agent: Google-Extended
Crawl-delay: 2
Allow: /

# Add your sitemap location
Sitemap: https://yourdomain.com/sitemap.xml