# Freedom Healthworks - The Operating System for DPC
# https://www.freedomhealthworks.com

# ===== BLOCKED CRAWLERS =====
# These bots are blocked because they pollute analytics or are not relevant to our US-based DPC market

# Chinese Search Engines & Crawlers
User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-image
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: ByteSpider
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# Russian Search Engines
User-agent: YandexBot
Disallow: /

User-agent: YandexImages
Disallow: /

User-agent: YandexMobileBot
Disallow: /

User-agent: YandexVideo
Disallow: /

# SEO/Marketing Crawlers (pollute analytics)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: Rogerbot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

# Spam/Scraper Bots
User-agent: MegaIndex
Disallow: /

User-agent: ltx71
Disallow: /

User-agent: BomboraBot
Disallow: /

User-agent: Buck
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: linkfluence
Disallow: /

# ===== ALLOWED CRAWLERS =====

# Search engines
User-agent: Googlebot
User-agent: Bingbot
Allow: /
Disallow: /thank-you
Disallow: /admin

User-agent: Googlebot-Image
Allow: /
Disallow: /admin

# Social link previews
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
Allow: /
Disallow: /thank-you
Disallow: /admin

# AI search and retrieval
# OpenAI treats search, user retrieval, and model-training controls separately.
User-agent: OAI-SearchBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: Anthropic-AI
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: Cohere-ai
User-agent: CCBot
Allow: /
Disallow: /thank-you
Disallow: /admin

# Default rule for unlisted bots
User-agent: *
Allow: /
Disallow: /thank-you
Disallow: /admin


# Crawl-delay for polite crawling
Crawl-delay: 1

# Sitemap
Sitemap: https://www.freedomhealthworks.com/sitemap.xml

# AI-Readable Content
# LLMs-Txt: https://www.freedomhealthworks.com/llms.txt