# =============================================
# robots.txt for tequila.ae
# Last updated: May 2026
# =============================================

# --- GLOBAL RULES (all bots) ---
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /?s=
Disallow: /*?s=
Disallow: /*&preview=
Disallow: /*?replytocom=
Disallow: /search/
Disallow: /author/
Disallow: /tag/
Disallow: /page/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-includes/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/

# =============================================
# OPENAI
# =============================================
# GPTBot       → model training
# OAI-SearchBot → ChatGPT search citations
# ChatGPT-User  → real-time user browsing

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# =============================================
# ANTHROPIC (Claude)
# =============================================
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# =============================================
# GOOGLE AI
# =============================================
# Google-Extended      → Gemini model training
# Gemini-Deep-Research → Gemini deep research
# Google-NotebookLM    → NotebookLM sources
# GoogleAgent-Mariner  → Google AI agent browser

User-agent: Google-Extended
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

User-agent: Google-NotebookLM
Allow: /

User-agent: GoogleAgent-Mariner
Allow: /

# =============================================
# PERPLEXITY
# =============================================
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# =============================================
# META / FACEBOOK
# =============================================
User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

# =============================================
# APPLE
# =============================================
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# =============================================
# AMAZON
# =============================================
User-agent: Amazonbot
Allow: /

# =============================================
# MICROSOFT / BING
# =============================================
User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

# =============================================
# OTHER MAJOR AI CRAWLERS
# =============================================
User-agent: YouBot
Allow: /

User-agent: Cohere-ai
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: TurnitinBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: omgili
Allow: /

User-agent: webzio-extended
Allow: /

User-agent: Kangaroo Bot
Allow: /

# =============================================
# SITEMAP
# =============================================
Sitemap: https://tequila.ae/sitemap_index.xml