{
  "study": "AI crawler robots.txt audit",
  "article": "https://chatgptgrow.com/research/gptbot-vs-oai-searchbot",
  "author": {
    "name": "Alex Li",
    "title": "Founder",
    "email": "founder@moeinsight.com",
    "url": "https://www.linkedin.com/in/alex-li-417b33417/"
  },
  "audited_at": "2026-09-15T06:40:20.901Z",
  "crawler_user_agent": "ChatgptGrowResearchBot/1.0 (+https://chatgptgrow.com/research/gptbot-vs-oai-searchbot)",
  "method": "GET https://<domain>/robots.txt; root-path evaluation only (Disallow: / or /* without Allow: / => blocked); bot-specific group takes precedence over *. Population B is a seeded random sample of the Tranco top-N list.",
  "tranco_sample": {
    "top_n": 50000,
    "size": 200,
    "seed": 20260915
  },
  "crawlers": [
    "GPTBot",
    "OAI-SearchBot",
    "ChatGPT-User",
    "OAI-AdsBot",
    "ClaudeBot",
    "PerplexityBot",
    "Google-Extended",
    "CCBot",
    "Bytespider",
    "Applebot-Extended",
    "meta-externalagent"
  ],
  "training_crawlers": [
    "GPTBot",
    "ClaudeBot",
    "CCBot",
    "Bytespider",
    "Google-Extended",
    "Applebot-Extended",
    "meta-externalagent"
  ],
  "retrieval_crawlers": [
    "OAI-SearchBot",
    "ChatGPT-User",
    "OAI-AdsBot",
    "PerplexityBot"
  ],
  "populations": {
    "vendor": {
      "domains_requested": 60,
      "robots_txt_served": 57,
      "description": "AI-visibility, GEO and AI-crawler tooling vendors. Used as a positive control on the parser rather than as a between-group finding: with n=57 the observed 0% is consistent with a true rate up to about 5%, which overlaps the general sample's interval."
    },
    "reference": {
      "domains_requested": 7,
      "robots_txt_served": 7,
      "domains": [
        "ahrefs.com",
        "chatgptgrow.com",
        "hubspot.com",
        "neilpatel.com",
        "openai.com",
        "searchengineland.com",
        "semrush.com"
      ],
      "description": "Hand-picked large reference sites, one of them our own. All seven block nothing. Not used as a finding."
    },
    "general": {
      "domains_requested": 200,
      "robots_txt_served": 110,
      "description": "Seeded random sample of the Tranco top 50,000 (seed 20260915). Dominated by large, platform-managed domains; it is not a sample of the web at large. Per-domain names are deliberately not published, but the sample is reproducible from the seed."
    }
  },
  "arithmetic": {
    "domains_requested": 267,
    "robots_txt_served": 174
  },
  "verdict_semantics": {
    "blocked": "The crawler's effective group contains a root disallow with no root allow.",
    "allowed": "An explicit root allow, a wildcard group with no root rule, no matching group, or a bot group with no root rule.",
    "unknown": "No robots.txt was served, so there was nothing to evaluate. These domains are excluded from every percentage below; every share is out of the population that served a file."
  },
  "results": {
    "vendor": {
      "domains_requested": 60,
      "robots_txt_served": 57,
      "blocks_any_training_crawler": 9,
      "blocks_any_retrieval_crawler": 0,
      "blocks_nothing": 48,
      "per_crawler_blocked": {
        "GPTBot": {
          "count": 3,
          "share_of_served": 0.0526
        },
        "OAI-SearchBot": {
          "count": 0,
          "share_of_served": 0.0
        },
        "ChatGPT-User": {
          "count": 0,
          "share_of_served": 0.0
        },
        "OAI-AdsBot": {
          "count": 0,
          "share_of_served": 0.0
        },
        "ClaudeBot": {
          "count": 3,
          "share_of_served": 0.0526
        },
        "PerplexityBot": {
          "count": 0,
          "share_of_served": 0.0
        },
        "Google-Extended": {
          "count": 4,
          "share_of_served": 0.0702
        },
        "CCBot": {
          "count": 7,
          "share_of_served": 0.1228
        },
        "Bytespider": {
          "count": 7,
          "share_of_served": 0.1228
        },
        "Applebot-Extended": {
          "count": 4,
          "share_of_served": 0.0702
        },
        "meta-externalagent": {
          "count": 4,
          "share_of_served": 0.0702
        }
      }
    },
    "reference": {
      "domains_requested": 7,
      "robots_txt_served": 7,
      "blocks_any_training_crawler": 0,
      "blocks_any_retrieval_crawler": 0,
      "blocks_nothing": 7,
      "per_crawler_blocked": {
        "GPTBot": {
          "count": 0,
          "share_of_served": 0.0
        },
        "OAI-SearchBot": {
          "count": 0,
          "share_of_served": 0.0
        },
        "ChatGPT-User": {
          "count": 0,
          "share_of_served": 0.0
        },
        "OAI-AdsBot": {
          "count": 0,
          "share_of_served": 0.0
        },
        "ClaudeBot": {
          "count": 0,
          "share_of_served": 0.0
        },
        "PerplexityBot": {
          "count": 0,
          "share_of_served": 0.0
        },
        "Google-Extended": {
          "count": 0,
          "share_of_served": 0.0
        },
        "CCBot": {
          "count": 0,
          "share_of_served": 0.0
        },
        "Bytespider": {
          "count": 0,
          "share_of_served": 0.0
        },
        "Applebot-Extended": {
          "count": 0,
          "share_of_served": 0.0
        },
        "meta-externalagent": {
          "count": 0,
          "share_of_served": 0.0
        }
      }
    },
    "general": {
      "domains_requested": 200,
      "robots_txt_served": 110,
      "blocks_any_training_crawler": 27,
      "blocks_any_retrieval_crawler": 10,
      "blocks_nothing": 83,
      "per_crawler_blocked": {
        "GPTBot": {
          "count": 21,
          "share_of_served": 0.1909
        },
        "OAI-SearchBot": {
          "count": 7,
          "share_of_served": 0.0636
        },
        "ChatGPT-User": {
          "count": 9,
          "share_of_served": 0.0818
        },
        "OAI-AdsBot": {
          "count": 4,
          "share_of_served": 0.0364
        },
        "ClaudeBot": {
          "count": 17,
          "share_of_served": 0.1545
        },
        "PerplexityBot": {
          "count": 8,
          "share_of_served": 0.0727
        },
        "Google-Extended": {
          "count": 17,
          "share_of_served": 0.1545
        },
        "CCBot": {
          "count": 23,
          "share_of_served": 0.2091
        },
        "Bytespider": {
          "count": 17,
          "share_of_served": 0.1545
        },
        "Applebot-Extended": {
          "count": 16,
          "share_of_served": 0.1455
        },
        "meta-externalagent": {
          "count": 17,
          "share_of_served": 0.1545
        }
      }
    }
  },
  "note": "Percentages are shares of the population that served a robots.txt, not of the population requested. Root-path evaluation only; this is not a complete robots.txt parser. Site configuration changes over time, so treat these as a snapshot of the audit date."
}