# HireFeed robots.txt — explicit AI-crawler allow-list.
# Per AEO best practice: name every AI crawler + Googlebot explicitly
# so nothing gets silently blocked by a default rule change.

# ChatGPT / OpenAI (training, live search, in-answer citation)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Claude / Anthropic (crawl + live search)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google Search + Google AI Overviews / Gemini
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Bing (ChatGPT pulls live results from Bing)
User-agent: Bingbot
Allow: /

# Common crawl (used as a base by many AI training pipelines)
User-agent: CCBot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: meta-externalagent
Allow: /

# Everyone else
User-agent: *
Allow: /

# Do not crawl the legacy backups + noindexed lead-gen pages
Disallow: /index-legacy
Disallow: /index-v8-pre-cosmic
Disallow: /popup-designs
Disallow: /_joblet_ref

Sitemap: https://hirefeed.co.in/sitemap.xml