# Generated 2026-04-18 by Claude Code master plan session
# PROPOSED robots.txt for discreet.toys
# Changes: B1 (parameter Disallow rules), B2 (nested feed Disallow),
#          B4 (2026-06-05 — expanded AI/LLM allowlist for discoverability)

# =====================================
# robots.txt for discreet.toys
# Updated: June 2026
# Lean SEO / WooCommerce / AI-discoverable version
# =====================================

Sitemap: https://discreet.toys/sitemap_index.xml

# =====================================
# Bots Completely Blocked
# =====================================
# 2026-06-05: REMOVED ClaudeBot + CCBot from this section.
# Both moved to the LLM/AI Bots section below to enable AI-assistant discoverability.
# Goal: be surfaced when users ask ChatGPT/Claude/Perplexity for adult wellness recommendations.

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: BlexBot
Disallow: /

# =====================================
# Facebook / Meta Crawlers (link-preview only)
# =====================================
# 2026-06-05: REMOVED meta-externalagent from this group — moved to LLM/AI section
# (it's Meta's AI training crawler, not the link-preview bot).
User-agent: Facebot
User-agent: Facebookexternalhit
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wp-json/
Disallow: /?wc-ajax=*
Disallow: /shop/
Disallow: /*?filter_*
Disallow: /*?min_price*
Disallow: /*?max_price*
Disallow: /*?orderby*

# =====================================
# Social Media Bots
# =====================================
User-agent: Twitterbot
User-agent: Pinterestbot
User-agent: LinkedInBot
User-agent: WhatsApp
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# =====================================
# LLM / AI Bots — EXPANDED 2026-06-05
# =====================================
# 2026-06-05: ADDED (B4) - Expanded AI bot allowlist for LLM discoverability.
# Replaced blanket "Disallow: /*?*" with the same hygiene pattern used for social bots,
# so AI crawlers can index product / category / pillar / blog content but skip
# cart funnel, admin, and faceted-filter noise.
# Bot UA strings to be re-verified per each vendor's published robots docs before deploy.

# --- OpenAI ---
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot

# --- Anthropic ---
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai

# --- Google (Gemini / Bard training opt-in) ---
User-agent: Google-Extended

# --- Apple Intelligence ---
User-agent: Applebot-Extended

# --- Meta AI ---
User-agent: Meta-ExternalAgent

# --- Amazon ---
User-agent: Amazonbot

# --- ByteDance / TikTok ---
User-agent: Bytespider

# --- DuckDuckGo Assist ---
User-agent: DuckAssistBot

# --- Perplexity ---
User-agent: PerplexityBot
User-agent: Perplexity-User

# --- Cohere ---
User-agent: cohere-ai

# --- Common Crawl (feeds most foundation-model training sets) ---
User-agent: CCBot

# --- Other ---
User-agent: Diffbot
User-agent: YouBot

Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /compare/
Disallow: /login/
Disallow: /register/
Disallow: /lost-password/
Disallow: /wc-api/
Disallow: /?wc-ajax=*
Disallow: /*?filter_features=
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?shop_view=
Disallow: /*?per_page=
Disallow: /*?per_row=
Disallow: /*?add-to-cart=
Disallow: /*?srsltid=
Disallow: /*?stock_status=
Disallow: /*?amp

Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/

# =====================================
# All Other Bots (Googlebot, Bingbot, etc.)
# =====================================
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /compare/
Disallow: /login/
Disallow: /register/
Disallow: /lost-password/
Disallow: /wc-api/
Disallow: /?wc-ajax=*
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /tag/
Disallow: /date/
Disallow: /search/
Disallow: /*/trackback/
Disallow: /*/attachment/*

# 2026-04-18: ADDED (B1) - Block faceted navigation and utility parameters for all bots.
# Prevents Googlebot/Bingbot from crawling parameterized filter/sort/price URLs.
# These generate thousands of duplicate/near-duplicate pages (10,905 alternate-with-canonical in GSC).
# Evidence: 06_SEO/redirects/REDIRECT_COVERAGE_MAP.md Priority Gap #2
Disallow: /*?filter_features=
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?shop_view=
Disallow: /*?per_page=
Disallow: /*?per_row=
Disallow: /*?add-to-cart=
Disallow: /*?amp

# 2026-04-23: ADDED (B3) - Block Google Shopping click-tracking + WooCommerce stock filter.
# Source: 10_ACTIONS/FINDINGS/2026-04-22_gsc_master_analysis.md section 4.3.2
# Evidence: 201 /shipping-return-policy/?srsltid=* URLs + ~30 /*?stock_status=* URLs in GSC failure reports.
Disallow: /*?srsltid=
Disallow: /*?stock_status=

# 2026-07-17: ADDED (B5) - Strategy A completion. Cover ALL WC filter facets (the old
# rule caught only filter_features; filter_color/size/material/... were crawlable and
# 403ing = the entire 2026-07 "403" bucket, 1,000+ URLs of pure crawl-budget burn).
# ?- and &-position variants included because /*?param= forms only match first-position
# params. Robots.txt is the SOLE param signal for Googlebot (Strategy A, Josh 2026-07-17).
Disallow: /*?filter_
Disallow: /*&filter_
Disallow: /*?query_type_
Disallow: /*&query_type_
Disallow: /*?orderby=
Disallow: /*&orderby=
Disallow: /*?post_type=
Disallow: /*&post_type=
Disallow: /*&min_price=
Disallow: /*&max_price=
Disallow: /*&add-to-cart=


# 2026-04-18: ADDED (B2) - Block nested feed URLs beyond root /feed/.
# Existing rule blocks /feed/ but not /product-name/feed/ or /brand/*/feed/.
# Evidence: 6 nested feed URLs found in GSC crawled-not-indexed sample.
Disallow: /*/feed/

Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/