# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
User-agent: *
Allow: /
# STRATEGY: Never use Disallow for pages that serve noindex.
# Blocking via robots.txt prevents Googlebot from seeing noindex → pages stay
# "Indexed, though blocked by robots.txt" indefinitely. Instead, let Googlebot
# crawl, read the noindex directive (meta tag or X-Robots-Tag header), and deindex.
#
# Paths that serve noindex and are NOT blocked here:
# every host page outside the current public pages → X-Robots-Tag via IndexPolicy
# (app/controllers/concerns/index_policy.rb), listed in sitemaps/retired.xml.gz
# /posts/*/drawer → 301 redirect to canonical post (non-turbo) + X-Robots-Tag
# /profile_entrypoint/ → 301 redirect to home (non-turbo) + X-Robots-Tag
# /rails/active_storage/ → X-Robots-Tag via initializer (active_storage_noindex.rb)
# /inscription/ → via devise_www layout
# /authentification → via devise_www layout
# /users/ → via devise_www layout
# /*?query=params → via SeoHelper
# /posts/page/ → via SeoHelper
# /posts/tag/ → via SeoHelper
#
# EXCEPTION — faceted search (/search/?by_sector=&by_region=&sort=&page=…):
# the ONE place we deliberately Disallow noindexed pages. This query-parameter
# space is ~1M near-duplicate combinations, all canonical-to-base and already
# "Crawled - currently not indexed" (never indexed) — so there is nothing to
# deindex and the usual "let Google see the noindex" rule does not apply.
# noindex does NOT stop crawling: Googlebot kept re-walking the combinatorial
# graph for months, starving the ~14k real pages of crawl budget. Blocking the
# crawl reclaims it. Curated /search//filter/:preset pages (no query
# string) stay crawlable and indexable; only the `?`-parameter variants are cut.
# Block authenticated/internal app routes (behind auth wall, no public content)
Disallow: /app/
Disallow: /system/
# Faceted-search query-parameter space (see EXCEPTION note above)
Disallow: /search/*?
# ============================================
# AI Search Crawlers — allowed for search visibility
# ============================================
User-agent: GPTBot
Allow: /
Disallow: /app/
Disallow: /system/
Disallow: /search/*?
User-agent: OAI-SearchBot
Allow: /
Disallow: /search/*?
User-agent: ClaudeBot
Allow: /
Disallow: /app/
Disallow: /system/
Disallow: /search/*?
User-agent: PerplexityBot
Allow: /
Disallow: /search/*?
User-agent: Google-Extended
Allow: /
Disallow: /search/*?
# ============================================
# Third-party SEO scrapers — blocked
# ============================================
# Backlink-index crawlers (Majestic, Moz). They send zero traffic and give us
# no Google / Bing / AI-chat visibility — pure crawl load. Also enforced with
# a hard 403 in app/middleware/bot_control.rb since they ignore robots.txt.
User-agent: MJ12bot
Disallow: /
User-agent: dotbot
Disallow: /
# ============================================
# Training-only crawlers — blocked
# ============================================
User-agent: CCBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: Amazonbot
Disallow: /
Sitemap: https://www.insiti.com/sitemaps/sitemap.xml.gz