User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /payment.html
Disallow: /admin.html
Disallow: /welcome.html
Disallow: /uninstall.html
Disallow: /thank-you.html

# Sitemap
Sitemap: https://ishort.pro/sitemap.xml
Sitemap: https://ishort.pro/sitemap.txt

# OpenAI Crawlers (https://developers.openai.com/api/docs/bots/)
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Anthropic / Claude Crawlers (https://support.claude.com/en/articles/8896518)
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Anthropic legacy tokens
User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Google AI (https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers)
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Perplexity (https://docs.perplexity.ai/guides/bots)
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Meta AI (https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)
User-agent: Meta-ExternalAgent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Common Crawl (used by many AI models)
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Cohere AI
User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# LLMs.txt for AI reference
# https://llmstxt.org/
Allow: /llms.txt