{
  "entity": {
    "id": 10,
    "slug": "googlebot",
    "vendor": "Google",
    "name": "Googlebot",
    "kind": "crawler",
    "purpose": "search",
    "ua_token": "Googlebot",
    "ua_pattern": "Googlebot",
    "robots_token": "Googlebot",
    "ip_list_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
    "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
    "first_documented": "2026-09-13",
    "first_seen_here": null,
    "last_seen_here": null,
    "status": "active",
    "notes": "Common crawler for Google Search, Discover, Images, Video and News.",
    "created_at": "2026-09-13T00:00:00Z",
    "updated_at": "2026-09-13T00:00:00Z"
  },
  "claims": [
    {
      "id": 149,
      "entity_id": 10,
      "slug": "googlebot/observed_verified_request_ratio_census",
      "field": "observed_verified_request_ratio_census",
      "value": "0.991",
      "statement": "Googlebot requests to Rattlesnakes By Mail matched Google's published IP ranges on 108 of 109 requests between 2026-09-13T20:00Z and 2026-09-15T09:00Z. Googlebot's first request to Rattlesnakes By Mail in that window was /robots.txt at 2026-09-14T11:53:00Z.",
      "evidence_url": "https://rattlesnakesbymail.com/observed/googlebot",
      "evidence_quote": "Googlebot: 109 requests, first 2026-09-14T11:53:00Z at /robots.txt; verified 108 of 109, 99.1%; formats html 68, txt 40, xml 1.",
      "method": "observed_here",
      "verified_at": "2026-09-15",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-17T21:31:41Z",
      "updated_at": "2026-09-17T21:31:41Z",
      "evidence_date": "2026-09-15"
    },
    {
      "id": 69,
      "entity_id": 10,
      "slug": "googlebot/cites_sources",
      "field": "cites_sources",
      "value": "yes",
      "statement": "Google AI Overviews and AI Mode return AI generated responses with links to supporting websites.",
      "evidence_url": "https://developers.google.com/search/docs/appearance/ai-features",
      "evidence_quote": "\"you can spend more time discovering ... get a comprehensive AI-powered response with links to supporting websites\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2025-12-10"
    },
    {
      "id": 64,
      "entity_id": 10,
      "slug": "googlebot/honours_crawl_delay",
      "field": "honours_crawl_delay",
      "value": "no",
      "statement": "Google does not support the robots.txt Crawl-delay directive for Googlebot.",
      "evidence_url": "https://developers.google.com/search/blog/2019/07/a-note-on-unsupported-rules-in-robotstxt",
      "evidence_quote": "\"Why isn't a code handler for other rules like crawl-delay included in the code?\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": null
    },
    {
      "id": 65,
      "entity_id": 10,
      "slug": "googlebot/publishes_ip_list",
      "field": "publishes_ip_list",
      "value": "yes",
      "statement": "Google publishes the IP address ranges used by Googlebot at https://developers.google.com/static/crawling/ipranges/common-crawlers.json.",
      "evidence_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "evidence_quote": "\"match the crawler's IP address to the lists of Google crawlers' and fetchers' IP ranges\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-09-11"
    },
    {
      "id": 67,
      "entity_id": 10,
      "slug": "googlebot/purpose_documented",
      "field": "purpose_documented",
      "value": "search",
      "statement": "Googlebot crawls pages for Google Search, Google Images, Google Video, Google News, and Discover.",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "evidence_quote": "\"Crawling preferences for Googlebot affect Google Search (including Discover and all Google Search features), as well as other products such as Google Images, Google Video, Google News, and Discover.\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-07-14"
    },
    {
      "id": 132,
      "entity_id": 10,
      "slug": "googlebot/reads_llms_txt",
      "field": "reads_llms_txt",
      "value": "no",
      "statement": "Google Search does not use llms.txt files.",
      "evidence_url": "https://developers.google.com/search/docs/fundamentals/ai-optimization-guide",
      "evidence_quote": "\"LLMS.txt files and other \"special\" markup: You don't need to create new machine readable files, AI text files, markup, or Markdown to appear in Google Search (including its generative AI capabilities), as Google Search itself doesn't use them.\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-14",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:32:29Z",
      "updated_at": "2026-09-14T00:32:29Z",
      "evidence_date": "2026-07-10"
    },
    {
      "id": 63,
      "entity_id": 10,
      "slug": "googlebot/respects_robots_txt",
      "field": "respects_robots_txt",
      "value": "yes",
      "statement": "Googlebot obeys robots.txt rules when crawling automatically.",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "evidence_quote": "\"They always obey robots.txt rules when crawling automatically.\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-07-14"
    },
    {
      "id": 68,
      "entity_id": 10,
      "slug": "googlebot/separate_search_and_training_tokens",
      "field": "separate_search_and_training_tokens",
      "value": "yes",
      "statement": "Google separates Search crawling under the Googlebot token from Gemini training under the Google-Extended token.",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "evidence_quote": "\"Google-Extended is a standalone product token that web publishers can use to manage whether content Google crawls from their sites may be used for training future generations of Gemini models\" and \"Google-Extended does not impact a site's inclusion in Google Search\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-07-14"
    },
    {
      "id": 70,
      "entity_id": 10,
      "slug": "googlebot/user_agent_string_full",
      "field": "user_agent_string_full",
      "value": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
      "statement": "Googlebot sends the user agent string Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html).",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "evidence_quote": null,
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-07-14"
    },
    {
      "id": 66,
      "entity_id": 10,
      "slug": "googlebot/verifiable_by_rdns",
      "field": "verifiable_by_rdns",
      "value": "yes",
      "statement": "Googlebot requests are verifiable by reverse DNS lookup resolving to googlebot.com, google.com, or googleusercontent.com.",
      "evidence_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verifying-googlebot",
      "evidence_quote": "\"Run a reverse DNS lookup on the accessing IP address from your logs, using the host command\" and \"Verify that the domain name is either googlebot.com, google.com, or googleusercontent.com.\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-03-20"
    }
  ],
  "observed": {
    "requests_30d": 1328,
    "verified_requests_30d": 628,
    "verified_share_30d": 0.47289156626506024,
    "daily_rows": [
      {
        "id": 155,
        "date": "2026-10-08",
        "entity_id": 10,
        "requests": 20,
        "paths": "[\"/robots.txt\",\"/fields/observed_first_request_path\",\"/fields/cites_sources\",\"/fields/publishes_ip_list\",\"/data\",\"/fields/anti_circumvention_stance\",\"/fields/robots_token_differs_from_ua\",\"/wordpress/\",\"/wp/\",\"/fields/fallback_to_googlebot_rules\",\"/fields/scope_of_crawling\",\"/\",\"/questions/d0b458517fb31268c7f8ec491094d483e38a6c87aa5c27109b3292365351b17c\"]",
        "formats": "[\"txt\",\"html\"]",
        "verified_share": 0.9,
        "verified_requests": 18
      },
      {
        "id": 150,
        "date": "2026-10-07",
        "entity_id": 10,
        "requests": 18,
        "paths": "[\"/.git/config\",\"/robots.txt\",\"/changes\",\"/data/latest/claims.json\",\"/crawlers/googleother\",\"/crawlers/claudebot\",\"/crawlers/oai-adsbot\",\"/crawlers/claude-user\",\"/\",\"/crawlers/chatgpt-user\"]",
        "formats": "[\"html\",\"txt\",\"json\"]",
        "verified_share": 0.9444444444444444,
        "verified_requests": 17
      },
      {
        "id": 144,
        "date": "2026-10-06",
        "entity_id": 10,
        "requests": 14,
        "paths": "[\"/crawlers/gptbot\",\"/robots.txt\",\"/claims\",\"/fields/robots_token_differs_from_ua\",\"/crawlers/google-extended\",\"/crawlers/claude-searchbot\"]",
        "formats": "[\"html\",\"txt\"]",
        "verified_share": 1,
        "verified_requests": 14
      },
      {
        "id": 138,
        "date": "2026-10-05",
        "entity_id": 10,
        "requests": 28,
        "paths": "[\"/observed\",\"/robots.txt\",\"/claims/82\",\"/questions\",\"/claims/12\",\"/\",\"/favicon.ico\",\"/crawlers/perplexitybot\",\"/observed/perplexitybot\",\"/observed/chatgpt-user\",\"/observed/bingbot\",\"/data/latest/observation_daily.csv\",\"/data/latest/claims.csv\",\"/data/latest/entities.csv\",\"/crawlers\"]",
        "formats": "[\"html\",\"txt\",\"csv\"]",
        "verified_share": 1,
        "verified_requests": 28
      },
      {
        "id": 133,
        "date": "2026-10-04",
        "entity_id": 10,
        "requests": 9,
        "paths": "[\"/robots.txt\",\"/data\",\"/crawlers/oai-searchbot\",\"/crawlers/googlebot\",\"/fields/publishes_ip_list\",\"/changes\"]",
        "formats": "[\"txt\",\"html\"]",
        "verified_share": 1,
        "verified_requests": 9
      },
      {
        "id": 123,
        "date": "2026-10-02",
        "entity_id": 10,
        "requests": 4,
        "paths": "[\"/data/latest/entities.json\",\"/robots.txt\",\"/sitemap.xml\"]",
        "formats": "[\"json\",\"txt\",\"xml\"]",
        "verified_share": 1,
        "verified_requests": 4
      },
      {
        "id": 118,
        "date": "2026-10-01",
        "entity_id": 10,
        "requests": 1,
        "paths": "[\"/robots.txt\"]",
        "formats": "[\"txt\"]",
        "verified_share": 1,
        "verified_requests": 1
      },
      {
        "id": 113,
        "date": "2026-09-30",
        "entity_id": 10,
        "requests": 7,
        "paths": "[\"/robots.txt\",\"/data/latest.json\",\"/questions/44b47f316eb91d7f8193930add0ac1a207a59de7fa38768c9f96d8b21fee11ba\",\"/crawlers/applebot-extended\",\"/compare/chatgpt-user-vs-perplexitybot\"]",
        "formats": "[\"txt\",\"json\",\"html\"]",
        "verified_share": 1,
        "verified_requests": 7
      },
      {
        "id": 108,
        "date": "2026-09-29",
        "entity_id": 10,
        "requests": 2,
        "paths": "[\"/robots.txt\",\"/questions/44b47f316eb91d7f8193930add0ac1a207a59de7fa38768c9f96d8b21fee11ba\"]",
        "formats": "[\"txt\",\"html\"]",
        "verified_share": 1,
        "verified_requests": 2
      },
      {
        "id": 102,
        "date": "2026-09-28",
        "entity_id": 10,
        "requests": 8,
        "paths": "[\"/robots.txt\",\"/crawlers/claudebot\",\"/data/latest/changes.csv\",\"/claims\",\"/crawlers/claude-user\"]",
        "formats": "[\"txt\",\"html\",\"csv\"]",
        "verified_share": 1,
        "verified_requests": 8
      },
      {
        "id": 95,
        "date": "2026-09-27",
        "entity_id": 10,
        "requests": 84,
        "paths": "[\"/robots.txt\",\"/method\",\"/claims/82\",\"/data/latest.json\",\"/questions\",\"/questions/62e5fac0b418c8f526b41170e57e65ea34a46a6278b2d876fc267034103664fc\",\"/questions/3008a9cd3da36dbd7db58b640d2b8c4f6e43e6a767ba573d5687bf39beb1bc64\",\"/\",\"/questions/28ab4ca1a97b70de471e253e4cc7a86059c59930cb5858e3465b35e48c71ff29\",\"/questions/97b24d1729c7f32e2e09cb297328a2f7977235c746960d243536ee8b9e4e494d\",\"/questions/7feb07db1e122a5d282af651c372b074a5316ec9f2a5b3bb7c6ed6f7e941ca59\",\"/questions/4fc955e1bb50d394bfad823b87d0b77f017ec05cb522fd90bdd876e7a4946a0b\",\"/questions/beb6bbb2d4693f277ef8dc453632449f11e9662a7ba33be712348f73944ce6d4\",\"/questions/79eecbe2d7bc2af29cfcce69b6efa8c4f7a0ddf030db007febb54b96ecf43744\",\"/questions/ff1c6deb90d74278593d6248167ee2547b98b46bb983ee68299165f27051e26e\",\"/questions/40d40a7e8b7733d1367a9185ae1bd0be486e49d8e0ce8db54a4fb3b737a9527e\",\"/questions/c1a2c20f142a7c2d2066ae6d41474c312396a2936a5ee55deb438dc71de7c81a\",\"/questions/e08c55e6af0f8858dae0af2af4eb4caafe28dfb739f1ed4163e9cb0d954bb2ff\",\"/questions/b906094064ac6f4a0b93a1e80f5c08f85bcc55d61e111a7470c0060db6b8a635\",\"/questions/3050a0ef31fe84bc2169c49e6513e17941f48a2c15f4720e45038792c2c8d29d\",\"/questions/263a3d5ba9e7e7d8ee555374ecf97a1bae8444842111226d90b1b3c2443761c1\",\"/questions/d0b458517fb31268c7f8ec491094d483e38a6c87aa5c27109b3292365351b17c\",\"/questions/ead6f2ea7befe0fc2648df1d7c8f962c9401177a05083ad897c5c091b77e478e\",\"/questions/fa5fdcff6b8a0f1f36241b515fbf6f51d5576026bbc4bd682d8d02f1c7460e1a\",\"/questions/04bc3496ca5e873179cdaa0304b17d77da22a5435c83472d0b62f386ed4c1533\",\"/questions/f8f07730d83f7ba170bf4d0154a86b78d920a5147e7ca0e03ca8ec8c80a59193\",\"/questions/1f586f71a0be68ee8ece2002622ee024a39cf4f46b0b8891d41f99a8626f5123\",\"/questions/c9bda3cd18b64dcd629f9501c7c4828409827a87b8cb7c3ebe39c6e0ed4fe655\",\"/questions/9ee0f4ebae4d73368ef6e2fcd2b9855fdf6193cea50f84f0836280acc79fb436\",\"/questions/a800346ce1243a685d6780ae2b5f91c373e30ceeb909dadca8f8eb1c84bfcf06\",\"/crawlers/chatgpt-user\",\"/questions/ae5ff38e6e254c7f33b7fcaeafdefdd7048c3656a7d00fde8995f34bd3a3d8ae\",\"/questions/6aa719abee602154bcfb38fc2c503493d4145501e25912e781b11faaa58f7b96\",\"/questions/19831deb9ab128884267ff7b26bb6fb86040e223b214ae652677f916039165f1\",\"/questions/b6282e25de55e809b2fc05b7b3c109a5d7029200175600460ce3a99c0ea2967e\",\"/questions/b1e41a4bbf179782994c46b459051bcdc8aa89f52103797e09588426375babdd\",\"/questions/ad86eb3ffc87efd1193f2c197e68578302af90fd56f3c3c47c71e1667afc68d6\",\"/questions/e61346be3fef727db8cefe0257815575a03a33698acfb1e0059140169dbd35ee\",\"/questions/4fd26a65bcb4f94d29d2a8ee92f5ec2208426425a69242efd5e4eb6f2daa84d6\",\"/questions/b35153b6fa9fcd738d70181548629cfc6af43dffe53d915d005f9434c55934e3\",\"/questions/51970f2a642188fdd2973fc74cc4ba88eb887493a2c7bf670b8fd84f326afbb5\",\"/questions/62283b8afc52091cd7cc5c2fbea34e6abe849012ec779dd4dfabd0553e908f92\",\"/questions/017c61ed27755bc71f9927e3dc1b9cedb6883f2e4cc44574a86c9aebb395fb14\",\"/questions/e907942e1bacc44659550c7e988ff69b89006901ce87a91e4889809bbf7c5909\",\"/questions/5d6dd57f58b6d5635f4b8879d7bbac4209829087fb894e9f82e6c25ac5729768\",\"/questions/1b9367705cf69e0948797407e3f39c731987989f41388f692a9d1a8eb8358f83\",\"/questions/064f0033dab1091575ab29137f558668ff6193e891221987fd9523c31b4ff78f\",\"/questions/988d882b4a74a8b8afb9f2d143bf2c538cecb7af4599cbaaa8260682d6aaf5b5\",\"/questions/3163fd879bdf5bb024eaf7d0d19ae5337f04474e1c6bc850ab124990d16e946d\",\"/questions/a86f29d86c0a795732314404684c110ab915c80ca08d0a4e4ec38527c642d974\",\"/questions/e091e35f0332f5c96f5d7a333659479e747c69c8c476931d742210bbbf0e557f\",\"/data\",\"/crawlers/googlebot\",\"/crawlers/claude-searchbot\",\"/fields/fallback_to_googlebot_rules\",\"/claims/17\"]",
        "formats": "[\"txt\",\"html\",\"json\"]",
        "verified_share": 1,
        "verified_requests": 84
      },
      {
        "id": 87,
        "date": "2026-09-26",
        "entity_id": 10,
        "requests": 12,
        "paths": "[\"/robots.txt\",\"/sitemap.xml\",\"/data/latest/changes.csv\",\"/crawlers/oai-adsbot\",\"/\",\"/crawlers\",\"/crawlers/google-extended\"]",
        "formats": "[\"txt\",\"xml\",\"csv\",\"html\"]",
        "verified_share": 1,
        "verified_requests": 12
      },
      {
        "id": 80,
        "date": "2026-09-25",
        "entity_id": 10,
        "requests": 21,
        "paths": "[\"/robots.txt\",\"/\",\"/crawlers/oai-searchbot\",\"/changes\",\"/crawlers/perplexitybot\",\"/compare/applebot-extended-vs-claude-user\"]",
        "formats": "[\"txt\",\"html\"]",
        "verified_share": 0.9523809523809523,
        "verified_requests": 20
      },
      {
        "id": 72,
        "date": "2026-09-24",
        "entity_id": 10,
        "requests": 9,
        "paths": "[\"/robots.txt\",\"/\",\"/favicon.ico\"]",
        "formats": "[\"txt\",\"html\"]",
        "verified_share": 1,
        "verified_requests": 9
      },
      {
        "id": 66,
        "date": "2026-09-23",
        "entity_id": 10,
        "requests": 28,
        "paths": "[\"/robots.txt\",\"/fields/anti_circumvention_stance\",\"/fields/robots_token_differs_from_ua\",\"/fields/observed_first_request_path\",\"/fields/scope_of_crawling\",\"/fields/fallback_to_googlebot_rules\",\"/fields/publishes_ip_list\"]",
        "formats": "[\"txt\",\"html\"]",
        "verified_share": 1,
        "verified_requests": 28
      },
      {
        "id": 60,
        "date": "2026-09-22",
        "entity_id": 10,
        "requests": 11,
        "paths": "[\"/robots.txt\",\"/sitemap.xml\",\"/fields/scope_of_crawling\"]",
        "formats": "[\"txt\",\"xml\",\"html\"]",
        "verified_share": 1,
        "verified_requests": 11
      },
      {
        "id": 54,
        "date": "2026-09-21",
        "entity_id": 10,
        "requests": 711,
        "paths": "[\"/robots.txt\",\"/fields/anti_circumvention_stance\",\"/fields/scope_of_crawling\",\"/fields/observed_first_request_path\",\"/data\",\"/\",\"/sitemap-index.xml\",\"/observed/claude-user\",\"/claims/61\",\"/observed\",\"/sitemap.xml\",\"/changes\",\"/sitemap-machine.xml\",\"/claims/59\",\"/claims/19\",\"/crawlers/googlebot\",\"/crawlers/google-extended\",\"/crawlers/bingbot\",\"/compare\",\"/crawlers/oai-adsbot\",\"/claims/12\",\"/observed/claudebot\",\"/claims/150\",\"/crawlers/chatgpt-user\",\"/claims/49\",\"/observed/googlebot\",\"/crawlers\",\"/questions/eaaa86558b443873916376dc4d0546f60ee5d890631698c41768fb619582451c\",\"/claims/47\",\"/claims\",\"/claims/82\",\"/claims/83\",\"/crawlers/claude-user\",\"/claims/148\",\"/claims/149\",\"/fields\",\"/claims/8\",\"/crawlers/claude-searchbot\",\"/crawlers/googleother\",\"/observed/applebot\",\"/crawlers/perplexity-user\",\"/method\",\"/questions\",\"/crawlers/claudebot\",\"/claims/132\",\"/observed/gptbot\",\"/claims/62\",\"/crawlers/applebot-extended\",\"/claims/66\",\"/claims/23\",\"/observed/googleother\",\"/crawlers/gptbot\",\"/crawlers/applebot\",\"/crawlers/oai-searchbot\",\"/claims/60\",\"/crawlers/perplexitybot\",\"/compare/chatgpt-user-vs-claudebot\",\"/claims/100\",\"/claims/101\",\"/claims/102\",\"/claims/51\",\"/claims/52\",\"/compare/google-extended-vs-perplexitybot\",\"/claims/53\",\"/claims/54\",\"/claims/55\",\"/claims/56\",\"/claims/57\",\"/claims/58\",\"/fields/honours_crawl_delay\",\"/claims/50\",\"/claims/110\",\"/claims/48\",\"/claims/111\",\"/compare/applebot-vs-chatgpt-user\",\"/claims/112\",\"/fields/observed_fetches_robots_txt\",\"/claims/113\",\"/claims/40\",\"/claims/41\",\"/claims/42\",\"/claims/43\",\"/claims/44\",\"/compare/bingbot-vs-oai-searchbot\",\"/claims/45\",\"/compare/claude-searchbot-vs-perplexity-user\",\"/claims/46\",\"/compare/applebot-vs-bingbot\",\"/compare/claude-user-vs-googleother\",\"/fields/verifiable_by_rdns\",\"/claims/103\",\"/claims/104\",\"/compare/bingbot-vs-perplexity-user\",\"/claims/105\",\"/claims/106\",\"/claims/107\",\"/claims/108\",\"/claims/109\",\"/claims/9\",\"/claims/4\"]",
        "formats": "[\"txt\",\"html\",\"xml\",\"csv\",\"md\"]",
        "verified_share": 0.02250351617440225,
        "verified_requests": 16
      },
      {
        "id": 50,
        "date": "2026-09-20",
        "entity_id": 10,
        "requests": 10,
        "paths": "[\"/robots.txt\",\"/data\",\"/fields/robots_token_differs_from_ua\",\"/fields/publishes_ip_list\",\"/fields/fallback_to_googlebot_rules\"]",
        "formats": "[\"txt\",\"html\"]",
        "verified_share": 1,
        "verified_requests": 10
      },
      {
        "id": 44,
        "date": "2026-09-19",
        "entity_id": 10,
        "requests": 7,
        "paths": "[\"/robots.txt\",\"/sitemap.xml\"]",
        "formats": "[\"txt\",\"xml\"]",
        "verified_share": 1,
        "verified_requests": 7
      },
      {
        "id": 37,
        "date": "2026-09-18",
        "entity_id": 10,
        "requests": 78,
        "paths": "[\"/robots.txt\",\"/sitemap.xml\",\"/fields/cites_sources\",\"/compare/gptbot-vs-perplexitybot\",\"/compare/claudebot-vs-oai-searchbot\",\"/fields/scope_of_crawling\",\"/fields/publishes_ip_list\",\"/claims/142\",\"/compare/gptbot-vs-perplexity-user\",\"/claims/128\",\"/compare/claudebot-vs-perplexitybot\",\"/claims/150\",\"/fields/observed_first_request_path\",\"/fields/fallback_to_googlebot_rules\",\"/fields/robots_token_differs_from_ua\",\"/claims/144\",\"/compare/oai-adsbot-vs-oai-searchbot\",\"/compare/claudebot-vs-perplexity-user\",\"/compare/claude-user-vs-oai-searchbot\",\"/compare/claude-searchbot-vs-oai-adsbot\",\"/claims/145\",\"/compare/applebot-extended-vs-oai-adsbot\",\"/claims/147\",\"/claims/143\",\"/compare/chatgpt-user-vs-perplexity-user\",\"/compare/claude-searchbot-vs-oai-searchbot\",\"/compare/claude-searchbot-vs-perplexitybot\",\"/compare/applebot-extended-vs-oai-searchbot\",\"/fields/anti_circumvention_stance\",\"/compare/applebot-extended-vs-claude-searchbot\",\"/questions/f453cacd1535f4700e95bab1b1480233e943644b1baa9857d70aa5001c264ee7\",\"/compare/claude-searchbot-vs-perplexity-user\",\"/compare/oai-searchbot-vs-perplexity-user\",\"/compare/oai-adsbot-vs-perplexity-user\",\"/compare/applebot-extended-vs-claude-user\"]",
        "formats": "[\"txt\",\"xml\",\"html\"]",
        "verified_share": 1,
        "verified_requests": 78
      },
      {
        "id": 29,
        "date": "2026-09-17",
        "entity_id": 10,
        "requests": 4,
        "paths": "[\"/robots.txt\"]",
        "formats": "[\"txt\"]",
        "verified_share": 1,
        "verified_requests": 4
      },
      {
        "id": 25,
        "date": "2026-09-16",
        "entity_id": 10,
        "requests": 30,
        "paths": "[\"/claims/32\",\"/robots.txt\",\"/data/latest/questions.csv\",\"/data\",\"/claims/66\",\"/claims/60\",\"/claims/69\",\"/claims/39\",\"/claims/44\",\"/claims/70\",\"/claims/68\",\"/claims/33\",\"/observed/applebot\",\"/observed/googlebot\",\"/observed/googleother\",\"/changes\",\"/questions/eaaa86558b443873916376dc4d0546f60ee5d890631698c41768fb619582451c\"]",
        "formats": "[\"html\",\"txt\",\"csv\"]",
        "verified_share": 1,
        "verified_requests": 30
      },
      {
        "id": 18,
        "date": "2026-09-15",
        "entity_id": 10,
        "requests": 133,
        "paths": "[\"/robots.txt\",\"/claims/3\",\"/crawlers/gptbot\",\"/claims/5\",\"/\",\"/claims/1\",\"/claims/2\",\"/claims/35\",\"/claims/127\",\"/questions\",\"/method\",\"/claims/91\",\"/observed/claudebot\",\"/changes\",\"/claims/36\",\"/crawlers/google-extended\",\"/claims/92\",\"/claims\",\"/claims/16\",\"/claims/37\",\"/claims/9\",\"/crawlers\",\"/claims/8\",\"/claims/94\",\"/claims/79\",\"/claims/111\",\"/claims/85\",\"/claims/7\",\"/claims/6\",\"/claims/21\",\"/claims/45\",\"/claims/57\",\"/claims/49\",\"/claims/90\",\"/claims/51\",\"/claims/34\",\"/claims/13\",\"/claims/62\",\"/claims/93\",\"/claims/11\",\"/claims/63\",\"/claims/96\",\"/claims/75\",\"/claims/46\",\"/claims/26\",\"/claims/77\",\"/claims/17\",\"/claims/28\",\"/claims/98\",\"/claims/18\",\"/claims/20\",\"/claims/19\",\"/claims/87\",\"/claims/95\",\"/claims/43\",\"/claims/10\",\"/claims/67\",\"/claims/73\",\"/claims/25\",\"/claims/29\",\"/claims/38\",\"/claims/30\",\"/claims/86\",\"/claims/15\",\"/claims/23\",\"/claims/65\",\"/claims/74\",\"/claims/99\",\"/claims/42\",\"/claims/109\",\"/claims/112\",\"/claims/132\",\"/claims/117\",\"/claims/110\",\"/claims/113\",\"/claims/102\",\"/claims/116\",\"/claims/114\",\"/claims/100\",\"/claims/101\",\"/claims/123\",\"/claims/120\",\"/claims/119\",\"/claims/118\",\"/claims/71\",\"/claims/104\",\"/sitemap.xml\",\"/claims/40\"]",
        "formats": "[\"txt\",\"html\",\"xml\"]",
        "verified_share": 1,
        "verified_requests": 133
      },
      {
        "id": 13,
        "date": "2026-09-14",
        "entity_id": 10,
        "requests": 79,
        "paths": "[\"/robots.txt\",\"/favicon.ico\",\"/\",\"/crawlers/google-extended\",\"/crawlers/applebot\",\"/crawlers/claudebot\",\"/changes\",\"/sitemap.xml\",\"/crawlers\",\"/claims\",\"/claims/12\",\"/observed\",\"/claims/88\",\"/data\",\"/method\",\"/claims/47\",\"/claims/48\",\"/claims/52\",\"/claims/61\",\"/claims/54\",\"/claims/53\",\"/claims/14\",\"/claims/82\",\"/claims/78\",\"/claims/55\",\"/claims/59\",\"/crawlers/googlebot\",\"/crawlers/claude-user\",\"/crawlers/googleother\",\"/crawlers/chatgpt-user\",\"/claims/108\",\"/crawlers/oai-searchbot\",\"/crawlers/claude-searchbot\",\"/crawlers/perplexity-user\",\"/crawlers/applebot-extended\",\"/claims/103\",\"/crawlers/oai-adsbot\",\"/claims/107\",\"/observed/oai-searchbot\",\"/crawlers/perplexitybot\",\"/observed/gptbot\",\"/claims/83\",\"/claims/105\"]",
        "formats": "[\"txt\",\"html\",\"xml\"]",
        "verified_share": 0.9873417721518988,
        "verified_requests": 78
      }
    ]
  },
  "changes": [
    {
      "id": 151,
      "claim_id": 149,
      "entity_id": 10,
      "changed_at": "2026-09-17T21:31:41Z",
      "kind": "new",
      "old_value": null,
      "new_value": "0.991",
      "evidence_url": "https://rattlesnakesbymail.com/observed/googlebot",
      "note": "observed_here claim from the September 2026 census (docs/census-2026-09.md), batch p1-observed-here-2026-09-17",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "observed_verified_request_ratio_census",
      "claim_value": "0.991",
      "claim_statement": "Googlebot requests to Rattlesnakes By Mail matched Google's published IP ranges on 108 of 109 requests between 2026-09-13T20:00Z and 2026-09-15T09:00Z. Googlebot's first request to Rattlesnakes By Mail in that window was /robots.txt at 2026-09-14T11:53:00Z."
    },
    {
      "id": 132,
      "claim_id": 132,
      "entity_id": 10,
      "changed_at": "2026-09-14T00:00:00Z",
      "kind": "new",
      "old_value": null,
      "new_value": "no",
      "evidence_url": "https://developers.google.com/search/docs/fundamentals/ai-optimization-guide",
      "note": "P0.3 googlebot llms.txt re-check batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "reads_llms_txt",
      "claim_value": "no",
      "claim_statement": "Google Search does not use llms.txt files."
    },
    {
      "id": 63,
      "claim_id": 63,
      "entity_id": 10,
      "changed_at": "2026-09-13T00:00:00Z",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "respects_robots_txt",
      "claim_value": "yes",
      "claim_statement": "Googlebot obeys robots.txt rules when crawling automatically."
    },
    {
      "id": 64,
      "claim_id": 64,
      "entity_id": 10,
      "changed_at": "2026-09-13T00:00:00Z",
      "kind": "new",
      "old_value": null,
      "new_value": "no",
      "evidence_url": "https://developers.google.com/search/blog/2019/07/a-note-on-unsupported-rules-in-robotstxt",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "honours_crawl_delay",
      "claim_value": "no",
      "claim_statement": "Google does not support the robots.txt Crawl-delay directive for Googlebot."
    },
    {
      "id": 65,
      "claim_id": 65,
      "entity_id": 10,
      "changed_at": "2026-09-13T00:00:00Z",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "publishes_ip_list",
      "claim_value": "yes",
      "claim_statement": "Google publishes the IP address ranges used by Googlebot at https://developers.google.com/static/crawling/ipranges/common-crawlers.json."
    },
    {
      "id": 66,
      "claim_id": 66,
      "entity_id": 10,
      "changed_at": "2026-09-13T00:00:00Z",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verifying-googlebot",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "verifiable_by_rdns",
      "claim_value": "yes",
      "claim_statement": "Googlebot requests are verifiable by reverse DNS lookup resolving to googlebot.com, google.com, or googleusercontent.com."
    },
    {
      "id": 67,
      "claim_id": 67,
      "entity_id": 10,
      "changed_at": "2026-09-13T00:00:00Z",
      "kind": "new",
      "old_value": null,
      "new_value": "search",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "purpose_documented",
      "claim_value": "search",
      "claim_statement": "Googlebot crawls pages for Google Search, Google Images, Google Video, Google News, and Discover."
    },
    {
      "id": 68,
      "claim_id": 68,
      "entity_id": 10,
      "changed_at": "2026-09-13T00:00:00Z",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "separate_search_and_training_tokens",
      "claim_value": "yes",
      "claim_statement": "Google separates Search crawling under the Googlebot token from Gemini training under the Google-Extended token."
    },
    {
      "id": 69,
      "claim_id": 69,
      "entity_id": 10,
      "changed_at": "2026-09-13T00:00:00Z",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/search/docs/appearance/ai-features",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "cites_sources",
      "claim_value": "yes",
      "claim_statement": "Google AI Overviews and AI Mode return AI generated responses with links to supporting websites."
    },
    {
      "id": 70,
      "claim_id": 70,
      "entity_id": 10,
      "changed_at": "2026-09-13T00:00:00Z",
      "kind": "new",
      "old_value": null,
      "new_value": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "claim_field": "user_agent_string_full",
      "claim_value": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
      "claim_statement": "Googlebot sends the user agent string Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html)."
    }
  ],
  "questions": [],
  "notes": [],
  "definition": "The Googlebot record on Rattlesnakes By Mail is the full published file on Googlebot, a crawler from Google. Each fact on the Googlebot record is a dated claim with a verbatim quote from Google, a source URL, a verification method and a confidence. Sections run in a fixed order: Identity, Claims, Observations, Changes, Questions, Notes.",
  "fields_url": "https://rattlesnakesbymail.com/fields",
  "compare_with": [
    {
      "slug": "claude-searchbot",
      "name": "Claude-SearchBot",
      "url": "https://rattlesnakesbymail.com/compare/claude-searchbot-vs-googlebot"
    },
    {
      "slug": "claude-user",
      "name": "Claude-User",
      "url": "https://rattlesnakesbymail.com/compare/claude-user-vs-googlebot"
    },
    {
      "slug": "claudebot",
      "name": "ClaudeBot",
      "url": "https://rattlesnakesbymail.com/compare/claudebot-vs-googlebot"
    },
    {
      "slug": "applebot",
      "name": "Applebot",
      "url": "https://rattlesnakesbymail.com/compare/applebot-vs-googlebot"
    },
    {
      "slug": "applebot-extended",
      "name": "Applebot-Extended",
      "url": "https://rattlesnakesbymail.com/compare/applebot-extended-vs-googlebot"
    },
    {
      "slug": "google-extended",
      "name": "Google-Extended",
      "url": "https://rattlesnakesbymail.com/compare/google-extended-vs-googlebot"
    },
    {
      "slug": "googleother",
      "name": "GoogleOther",
      "url": "https://rattlesnakesbymail.com/compare/googlebot-vs-googleother"
    },
    {
      "slug": "bingbot",
      "name": "Bingbot",
      "url": "https://rattlesnakesbymail.com/compare/bingbot-vs-googlebot"
    },
    {
      "slug": "chatgpt-user",
      "name": "ChatGPT-User",
      "url": "https://rattlesnakesbymail.com/compare/chatgpt-user-vs-googlebot"
    },
    {
      "slug": "gptbot",
      "name": "GPTBot",
      "url": "https://rattlesnakesbymail.com/compare/googlebot-vs-gptbot"
    },
    {
      "slug": "oai-adsbot",
      "name": "OAI-AdsBot",
      "url": "https://rattlesnakesbymail.com/compare/googlebot-vs-oai-adsbot"
    },
    {
      "slug": "oai-searchbot",
      "name": "OAI-SearchBot",
      "url": "https://rattlesnakesbymail.com/compare/googlebot-vs-oai-searchbot"
    },
    {
      "slug": "perplexity-user",
      "name": "Perplexity-User",
      "url": "https://rattlesnakesbymail.com/compare/googlebot-vs-perplexity-user"
    },
    {
      "slug": "perplexitybot",
      "name": "PerplexityBot",
      "url": "https://rattlesnakesbymail.com/compare/googlebot-vs-perplexitybot"
    }
  ]
}