User-agent: *
Allow: /

# Disallow internal/authenticated pages
Disallow: /api/
Disallow: /auth/
Disallow: /processing/
Disallow: /transcribe/
Disallow: /*/transcribe/
Disallow: /completion/
Disallow: /return/
Disallow: /transcriptions
Disallow: /collections
Disallow: /new
Disallow: /account
Disallow: /payment
Disallow: /library/search
Disallow: /giraffe
Disallow: /cheetah
Disallow: /_seo/

# Link-preview (unfurl) crawlers - same restrictions as *, except /transcribe/
# is ALLOWED so shared score links render a card on X / Facebook / Slack / etc.
# Twitterbot honours robots.txt and silently drops the card when the URL is
# disallowed. The pages themselves stay out of search via a noindex meta tag.
# NOTE: crawlers matching this group IGNORE the * group above entirely —
# any new Disallow added to the * group must be duplicated here too.
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Slackbot-LinkExpanding
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /processing/
Disallow: /completion/
Disallow: /return/
Disallow: /transcriptions
Disallow: /collections
Disallow: /new
Disallow: /account
Disallow: /payment
Disallow: /library/search
Disallow: /giraffe
Disallow: /cheetah
Disallow: /_seo/

# AI search crawlers - explicitly allowed (same path restrictions as *).
# NOTE: crawlers matching this group IGNORE the * group above entirely —
# any new Disallow added to the * group must be duplicated here too.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: CCBot
User-agent: meta-webindexer
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: GrokBot
User-agent: xAI-Grok
User-agent: Grok-DeepSearch
User-agent: xAI-Bot
User-agent: Grok
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /processing/
Disallow: /transcribe/
Disallow: /*/transcribe/
Disallow: /completion/
Disallow: /return/
Disallow: /transcriptions
Disallow: /collections
Disallow: /new
Disallow: /account
Disallow: /payment
Disallow: /library/search
Disallow: /giraffe
Disallow: /cheetah
Disallow: /_seo/

# Sitemap location
Sitemap: https://www.songscription.ai/sitemap.xml