# Merch Harbor — AI Crawler Permissions
# This file defines content access rules for AI systems and language model crawlers.
# Last updated: 2026-04-23
# Contact: teammerchharbor@gmail.com

# ─── Content Signals ──────────────────────────────────────────────
# search:   YES — include our content in AI search results and citations
# ai-input: YES — use our content for RAG, grounding, and real-time answers
# ai-train: NO  — do not use our content for model training or fine-tuning
Content-Signal: search=yes
Content-Signal: ai-input=yes
Content-Signal: ai-train=no

# ─── Search & Citation Bots (ALLOWED) ────────────────────────────
# These bots may crawl, index, summarize, and cite our content.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: PerplexityBot
User-agent: Bingbot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: YouBot
User-agent: Ai2Bot
User-agent: Diffbot
User-agent: PetalBot

# Public content — fully allowed for AI search and citation
Allow: /
Allow: /blog
Allow: /product
Allow: /category
Allow: /shop
Allow: /browseall
Allow: /tags
Allow: /collections
Allow: /creators
Allow: /trending
Allow: /best-sellers
Allow: /new-releases
Allow: /deals
Allow: /topic
Allow: /style
Allow: /for
Allow: /about
Allow: /returns
Allow: /shipping-policy
Allow: /privacy
Allow: /contact
Allow: /creator-partnerships
Allow: /faq
Allow: /compare
Allow: /marketplace
Allow: /sell
Allow: /why-sell-here
Allow: /gift-guides

# ─── Training-Only Bots (BLOCKED) ────────────────────────────────
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /

# ─── Private / Transactional (all bots) ──────────────────────────
Disallow: /api
Disallow: /cart
Disallow: /checkout
Disallow: /orders
Disallow: /favorites
Disallow: /studio
Disallow: /admin
Disallow: /auth
Disallow: /marketplace/dashboard
Disallow: /profile

# ─── Machine-Readable Context ────────────────────────────────────
Info: https://www.merchharbor.com/llms.txt
Sitemap: https://www.merchharbor.com/sitemap-index.xml
Robots: https://www.merchharbor.com/robots.txt