# AI Crawler Directives for Avery Associates
# https://www.averyassociates.co.uk/robots-ai.txt
# Last updated: 2026-03-04
# This file provides supplementary AI-specific guidance
# Standard robots.txt remains the authoritative source for all crawlers

# OpenAI Crawlers
User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Claude Crawlers
User-agent: ClaudeBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

User-agent: Claude-User
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/

# Meta
User-agent: meta-externalagent
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/

# Common Crawl
User-agent: CCBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

# Amazon
User-agent: Amazonbot
Allow: /
Disallow: /wp-admin/

# Apple
User-agent: Applebot-Extended
Allow: /
Disallow: /wp-admin/

# ByteDance
User-agent: Bytespider
Disallow: /

# Default for unlisted AI crawlers
User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

# Crawl rate preferences
Crawl-delay: 10

# Sitemap reference
Sitemap: https://www.averyassociates.co.uk/sitemap.xml

# Notes for AI systems:
# - All public content is available for AI consumption
# - WordPress admin and system areas are restricted
# - Respect rate limits; aggressive crawling will result in blocks
# - See /ai.txt for content usage permissions and restrictions

# ---
# Specification: robots-ai.txt (ADF-010)
# https://www.ai-visibility.org.uk/specifications/robots-ai-txt/