# Robots.txt for forsocials.com
# Allows AI agents and legitimate crawlers while protecting against scrapers.

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: GPTBot
Allow: /
# GPTBot = OpenAI’s crawler for training models and AI agents

User-agent: anthropic-ai
Allow: /
# Claude AI’s crawler

User-agent: PerplexityBot
Allow: /
# Perplexity AI

User-agent: DuckDuckBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: CCBot
Allow: /
# Common Crawl (used by many AI datasets)

# Disallow generic scrapers and bots
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: *
Disallow: /admin
Disallow: /api
Disallow: /SecondaryCTA
Disallow: /welcome-to-docs
Disallow: /*/welcome-to-docs
Allow: /

Sitemap: https://www.forsocials.com/sitemap-index.xml