User-agent: *
Allow: /
Allow: /wp-content/plugins/*.js
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.jpg
Allow: /wp-content/plugins/*.jpeg
Allow: /wp-content/plugins/*.png
Allow: /wp-content/plugins/*.gif
Allow: /wp-content/plugins/*.woff
Allow: /wp-content/plugins/*.ttf
Disallow: /wp-admin
Disallow: /wp-content/plugins
Disallow: /thank-you/
Disallow: /cdn-cgi/

# Major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# AI assistants and chatbots — allow content indexing and citation
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Google AI training and Gemini — allow inference, allow
User-agent: Google-Extended
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: meta-externalagent
Allow: /

# Microsoft Copilot / Bing AI
User-agent: bingbot
Allow: /

# Common Crawl — used by many AI training datasets
User-agent: CCBot
Allow: /

# ByteDance / TikTok AI — disallow (aggressive crawler, not aligned with education)
User-agent: Bytespider
Disallow: /

# AI.txt location hint (see /ai.txt for AI-specific access preferences)
# LLMs.txt location (see /llms.txt for curated LLM-readable content)
# LLMs-full.txt location (see /llms-full.txt for complete site content)

# Content Signals for AI (https://contentsignals.org/ / draft-romm-aipref-contentsignals)
# ai-train=no  : do not use our content to train AI models
# search=yes   : allow search-engine indexing
# ai-input=yes : AI agents may use our content to answer user questions about AVI
Content-Signal: ai-train=no, search=yes, ai-input=yes

Sitemap: https://avi.edu/sitemap_index.xml