# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file User-agent: * Allow: / # STRATEGY: Never use Disallow for pages that serve noindex. # Blocking via robots.txt prevents Googlebot from seeing noindex → pages stay # "Indexed, though blocked by robots.txt" indefinitely. Instead, let Googlebot # crawl, read the noindex directive (meta tag or X-Robots-Tag header), and deindex. # # Paths that serve noindex and are NOT blocked here: # every host page outside the current public pages → X-Robots-Tag via IndexPolicy # (app/controllers/concerns/index_policy.rb), listed in sitemaps/retired.xml.gz # /posts/*/drawer → 301 redirect to canonical post (non-turbo) + X-Robots-Tag # /profile_entrypoint/ → 301 redirect to home (non-turbo) + X-Robots-Tag # /rails/active_storage/ → X-Robots-Tag via initializer (active_storage_noindex.rb) # /inscription/ → via devise_www layout # /authentification → via devise_www layout # /users/ → via devise_www layout # /*?query=params → via SeoHelper # /posts/page/ → via SeoHelper # /posts/tag/ → via SeoHelper # # EXCEPTION — faceted search (/search/?by_sector=&by_region=&sort=&page=…): # the ONE place we deliberately Disallow noindexed pages. This query-parameter # space is ~1M near-duplicate combinations, all canonical-to-base and already # "Crawled - currently not indexed" (never indexed) — so there is nothing to # deindex and the usual "let Google see the noindex" rule does not apply. # noindex does NOT stop crawling: Googlebot kept re-walking the combinatorial # graph for months, starving the ~14k real pages of crawl budget. Blocking the # crawl reclaims it. Curated /search//filter/:preset pages (no query # string) stay crawlable and indexable; only the `?`-parameter variants are cut. # Block authenticated/internal app routes (behind auth wall, no public content) Disallow: /app/ Disallow: /system/ # Faceted-search query-parameter space (see EXCEPTION note above) Disallow: /search/*? # ============================================ # AI Search Crawlers — allowed for search visibility # ============================================ User-agent: GPTBot Allow: / Disallow: /app/ Disallow: /system/ Disallow: /search/*? User-agent: OAI-SearchBot Allow: / Disallow: /search/*? User-agent: ClaudeBot Allow: / Disallow: /app/ Disallow: /system/ Disallow: /search/*? User-agent: PerplexityBot Allow: / Disallow: /search/*? User-agent: Google-Extended Allow: / Disallow: /search/*? # ============================================ # Third-party SEO scrapers — blocked # ============================================ # Backlink-index crawlers (Majestic, Moz). They send zero traffic and give us # no Google / Bing / AI-chat visibility — pure crawl load. Also enforced with # a hard 403 in app/middleware/bot_control.rb since they ignore robots.txt. User-agent: MJ12bot Disallow: / User-agent: dotbot Disallow: / # ============================================ # Training-only crawlers — blocked # ============================================ User-agent: CCBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Amazonbot Disallow: / Sitemap: https://www.insiti.com/sitemaps/sitemap.xml.gz