# robots.txt for starsling.dev
#
# StarSling is an AI-native company: we WANT to be crawled, indexed, and cited.
# Only /dashboard (the authenticated app) is disallowed; everything else is open.

# --- Retrieval / search bots: ALLOW (live citations + search ranking) ---
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard

User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard

User-agent: Claude-User
Allow: /
Disallow: /dashboard

User-agent: Claude-SearchBot
Allow: /
Disallow: /dashboard

User-agent: PerplexityBot
Allow: /
Disallow: /dashboard

User-agent: Perplexity-User
Allow: /
Disallow: /dashboard

User-agent: Googlebot
Allow: /
Disallow: /dashboard

User-agent: Bingbot
Allow: /
Disallow: /dashboard

# --- Training / dataset bots: ALLOW (deliberate IP tradeoff; see notes) ---
User-agent: GPTBot
Allow: /
Disallow: /dashboard

User-agent: Google-Extended
Allow: /
Disallow: /dashboard

User-agent: ClaudeBot
Allow: /
Disallow: /dashboard

User-agent: anthropic-ai
Allow: /
Disallow: /dashboard

User-agent: Claude-Web
Allow: /
Disallow: /dashboard

User-agent: CCBot
Allow: /
Disallow: /dashboard

# --- Everyone else ---
# Cloudflare Content Signals (https://contentsignals.org):
#   search=yes   -> allow building search indexes
#   ai-input=yes -> allow AI agents to use this content as task input
#   ai-train=yes -> allow training of AI models on this content
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /dashboard

Sitemap: https://starsling.dev/sitemap.xml

# AI/LLM-friendly content available at:
# /llms.txt - Lean index + disambiguation for language models
# /llms-full.txt - Full reference (runner labels, pricing, limitations, setup)
# /index.md - Home page in Markdown
# /blog.md - Blog post index in Markdown
# /blog/<slug>.md - Each post in Markdown
# /ai-assistant-facts.md - AI assistant facts in Markdown
# /ai-native-ci.md - AI-native CI explainer in Markdown
# /compare.md and /compare/<slug>.md - comparison pages in Markdown
#
# Feeds:
# /rss.xml - Atom 1.0 feed (full content)
# /feed.json - JSON Feed 1.1 (full content)