{
  "day": "2026-08-09",
  "generatedAt": "2026-08-09T18:26:24.587Z",
  "vantage": "gha-ubuntu",
  "versions": {
    "probe": "2.0.0",
    "rubric": "1.0.0",
    "registry": "1.0.0"
  },
  "domainsIndexed": 4984,
  "domainsScored": 3633,
  "meanScore": 64.43,
  "blockingAnyAnswerSurfaceCrawler": 651,
  "blockingAllAnswerSurfaceCrawlers": 157,
  "publishingLlmsTxt": 445,
  "publishingAgentsMd": 28,
  "refusedGptbotAtTheServer": 811,
  "chargingForCrawlAccess": 11,
  "blockedByCrawler": [
    {
      "token": "GPTBot",
      "operator": "OpenAI",
      "tier": 1,
      "blockedBy": 547
    },
    {
      "token": "OAI-SearchBot",
      "operator": "OpenAI",
      "tier": 1,
      "blockedBy": 260
    },
    {
      "token": "ChatGPT-User",
      "operator": "OpenAI",
      "tier": 1,
      "blockedBy": 321
    },
    {
      "token": "ClaudeBot",
      "operator": "Anthropic",
      "tier": 1,
      "blockedBy": 522
    },
    {
      "token": "Claude-User",
      "operator": "Anthropic",
      "tier": 1,
      "blockedBy": 266
    },
    {
      "token": "Claude-SearchBot",
      "operator": "Anthropic",
      "tier": 1,
      "blockedBy": 260
    },
    {
      "token": "PerplexityBot",
      "operator": "Perplexity",
      "tier": 1,
      "blockedBy": 376
    },
    {
      "token": "Perplexity-User",
      "operator": "Perplexity",
      "tier": 1,
      "blockedBy": 273
    },
    {
      "token": "Google-Extended",
      "operator": "Google",
      "tier": 1,
      "blockedBy": 473
    },
    {
      "token": "Applebot-Extended",
      "operator": "Apple",
      "tier": 1,
      "blockedBy": 447
    },
    {
      "token": "meta-externalagent",
      "operator": "Meta",
      "tier": 1,
      "blockedBy": 485
    },
    {
      "token": "CCBot",
      "operator": "Common Crawl",
      "tier": 2,
      "blockedBy": 574
    },
    {
      "token": "Amazonbot",
      "operator": "Amazon",
      "tier": 2,
      "blockedBy": 439
    },
    {
      "token": "Bytespider",
      "operator": "ByteDance",
      "tier": 2,
      "blockedBy": 538
    },
    {
      "token": "cohere-ai",
      "operator": "Cohere",
      "tier": 2,
      "blockedBy": 378
    },
    {
      "token": "MistralAI-User",
      "operator": "Mistral",
      "tier": 2,
      "blockedBy": 258
    },
    {
      "token": "YouBot",
      "operator": "You.com",
      "tier": 2,
      "blockedBy": 327
    },
    {
      "token": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "tier": 2,
      "blockedBy": 274
    },
    {
      "token": "kagi-fetcher",
      "operator": "Kagi",
      "tier": 2,
      "blockedBy": 167
    },
    {
      "token": "Diffbot",
      "operator": "Diffbot",
      "tier": 2,
      "blockedBy": 375
    },
    {
      "token": "Google-NotebookLM",
      "operator": "Google",
      "tier": 2,
      "blockedBy": 188
    },
    {
      "token": "TavilyBot",
      "operator": "Tavily",
      "tier": 2,
      "blockedBy": 197
    },
    {
      "token": "FirecrawlAgent",
      "operator": "Firecrawl",
      "tier": 2,
      "blockedBy": 206
    }
  ],
  "byNetwork": [
    {
      "id": "cloudflare",
      "sites": 1082,
      "blockingAnyAnswerSurfaceCrawler": 187,
      "blockingRatePercent": 17.28,
      "meanScore": 72
    },
    {
      "id": "cloudfront",
      "sites": 413,
      "blockingAnyAnswerSurfaceCrawler": 82,
      "blockingRatePercent": 19.85,
      "meanScore": 64.2
    },
    {
      "id": "akamai",
      "sites": 265,
      "blockingAnyAnswerSurfaceCrawler": 43,
      "blockingRatePercent": 16.23,
      "meanScore": 69.7
    },
    {
      "id": "fastly",
      "sites": 216,
      "blockingAnyAnswerSurfaceCrawler": 53,
      "blockingRatePercent": 24.54,
      "meanScore": 66.3
    },
    {
      "id": "google",
      "sites": 153,
      "blockingAnyAnswerSurfaceCrawler": 5,
      "blockingRatePercent": 3.27,
      "meanScore": 57.8
    },
    {
      "id": "vercel",
      "sites": 48,
      "blockingAnyAnswerSurfaceCrawler": 3,
      "blockingRatePercent": 6.25,
      "meanScore": 77.2
    },
    {
      "id": "varnish",
      "sites": 39,
      "blockingAnyAnswerSurfaceCrawler": 12,
      "blockingRatePercent": 30.77,
      "meanScore": 60.4
    },
    {
      "id": "alibaba",
      "sites": 37,
      "blockingAnyAnswerSurfaceCrawler": 5,
      "blockingRatePercent": 13.51,
      "meanScore": 52.7
    },
    {
      "id": "azure-frontdoor",
      "sites": 28,
      "blockingAnyAnswerSurfaceCrawler": 1,
      "blockingRatePercent": 3.57,
      "meanScore": 62.4
    },
    {
      "id": "netlify",
      "sites": 25,
      "blockingAnyAnswerSurfaceCrawler": 1,
      "blockingRatePercent": 4,
      "meanScore": 73.7
    }
  ],
  "byPlatform": [
    {
      "id": "wordpress",
      "sites": 300,
      "blockingAnyAnswerSurfaceCrawler": 52,
      "blockingRatePercent": 17.33,
      "meanScore": 72.3
    },
    {
      "id": "nextjs",
      "sites": 280,
      "blockingAnyAnswerSurfaceCrawler": 43,
      "blockingRatePercent": 15.36,
      "meanScore": 68.4
    },
    {
      "id": "aem",
      "sites": 130,
      "blockingAnyAnswerSurfaceCrawler": 6,
      "blockingRatePercent": 4.62,
      "meanScore": 68.6
    },
    {
      "id": "drupal",
      "sites": 114,
      "blockingAnyAnswerSurfaceCrawler": 2,
      "blockingRatePercent": 1.75,
      "meanScore": 66.1
    },
    {
      "id": "hubspot",
      "sites": 80,
      "blockingAnyAnswerSurfaceCrawler": 3,
      "blockingRatePercent": 3.75,
      "meanScore": 74.6
    },
    {
      "id": "nuxt",
      "sites": 57,
      "blockingAnyAnswerSurfaceCrawler": 4,
      "blockingRatePercent": 7.02,
      "meanScore": 64.3
    },
    {
      "id": "contentful",
      "sites": 50,
      "blockingAnyAnswerSurfaceCrawler": 1,
      "blockingRatePercent": 2,
      "meanScore": 70.6
    },
    {
      "id": "sanity",
      "sites": 30,
      "blockingAnyAnswerSurfaceCrawler": 3,
      "blockingRatePercent": 10,
      "meanScore": 78.7
    }
  ],
  "byTld": [
    {
      "id": "com",
      "sites": 1948,
      "blockingAnyAnswerSurfaceCrawler": 374,
      "blockingRatePercent": 19.2,
      "meanScore": 66.4
    },
    {
      "id": "org",
      "sites": 215,
      "blockingAnyAnswerSurfaceCrawler": 23,
      "blockingRatePercent": 10.7,
      "meanScore": 66.4
    },
    {
      "id": "net",
      "sites": 182,
      "blockingAnyAnswerSurfaceCrawler": 37,
      "blockingRatePercent": 20.33,
      "meanScore": 60.4
    },
    {
      "id": "ru",
      "sites": 145,
      "blockingAnyAnswerSurfaceCrawler": 13,
      "blockingRatePercent": 8.97,
      "meanScore": 61.5
    },
    {
      "id": "edu",
      "sites": 81,
      "blockingAnyAnswerSurfaceCrawler": 5,
      "blockingRatePercent": 6.17,
      "meanScore": 66.4
    },
    {
      "id": "io",
      "sites": 74,
      "blockingAnyAnswerSurfaceCrawler": 6,
      "blockingRatePercent": 8.11,
      "meanScore": 68
    },
    {
      "id": "de",
      "sites": 71,
      "blockingAnyAnswerSurfaceCrawler": 21,
      "blockingRatePercent": 29.58,
      "meanScore": 61.3
    },
    {
      "id": "gov",
      "sites": 66,
      "blockingAnyAnswerSurfaceCrawler": 3,
      "blockingRatePercent": 4.55,
      "meanScore": 66.8
    },
    {
      "id": "jp",
      "sites": 34,
      "blockingAnyAnswerSurfaceCrawler": 5,
      "blockingRatePercent": 14.71,
      "meanScore": 62.2
    },
    {
      "id": "fr",
      "sites": 32,
      "blockingAnyAnswerSurfaceCrawler": 18,
      "blockingRatePercent": 56.25,
      "meanScore": 56.8
    },
    {
      "id": "com.br",
      "sites": 30,
      "blockingAnyAnswerSurfaceCrawler": 4,
      "blockingRatePercent": 13.33,
      "meanScore": 68.4
    },
    {
      "id": "tv",
      "sites": 27,
      "blockingAnyAnswerSurfaceCrawler": 3,
      "blockingRatePercent": 11.11,
      "meanScore": 57.3
    },
    {
      "id": "it",
      "sites": 26,
      "blockingAnyAnswerSurfaceCrawler": 10,
      "blockingRatePercent": 38.46,
      "meanScore": 62.1
    },
    {
      "id": "co.uk",
      "sites": 25,
      "blockingAnyAnswerSurfaceCrawler": 14,
      "blockingRatePercent": 56,
      "meanScore": 52.4
    },
    {
      "id": "cn",
      "sites": 21,
      "blockingAnyAnswerSurfaceCrawler": 1,
      "blockingRatePercent": 4.76,
      "meanScore": 53
    },
    {
      "id": "me",
      "sites": 19,
      "blockingAnyAnswerSurfaceCrawler": 1,
      "blockingRatePercent": 5.26,
      "meanScore": 64.9
    }
  ],
  "history": [
    {
      "day": "2026-08-09",
      "totalDomains": 4984,
      "observed": 3633,
      "meanScore": 64.43,
      "blockingAnyTier1": 651,
      "blockingAllTier1": 157,
      "llmsTxt": 445,
      "agentsMd": 28,
      "refusedGptbot": 811,
      "paymentRequired": 11,
      "perBot": {
        "GPTBot": 547,
        "OAI-SearchBot": 260,
        "ChatGPT-User": 321,
        "ClaudeBot": 522,
        "Claude-User": 266,
        "Claude-SearchBot": 260,
        "PerplexityBot": 376,
        "Perplexity-User": 273,
        "Google-Extended": 473,
        "Applebot-Extended": 447,
        "meta-externalagent": 485,
        "CCBot": 574,
        "Amazonbot": 439,
        "Bytespider": 538,
        "cohere-ai": 378,
        "MistralAI-User": 258,
        "YouBot": 327,
        "DuckAssistBot": 274,
        "kagi-fetcher": 167,
        "Diffbot": 375,
        "Google-NotebookLM": 188,
        "TavilyBot": 197,
        "FirecrawlAgent": 206
      },
      "perPlatform": {
        "nextjs": {
          "total": 280,
          "blocking": 43
        },
        "wordpress": {
          "total": 300,
          "blocking": 52
        },
        "contentful": {
          "total": 50,
          "blocking": 1
        },
        "aem": {
          "total": 130,
          "blocking": 6
        },
        "nuxt": {
          "total": 57,
          "blocking": 4
        },
        "gatsby": {
          "total": 11,
          "blocking": 1
        },
        "sitecore": {
          "total": 9,
          "blocking": 0
        },
        "hubspot": {
          "total": 80,
          "blocking": 3
        },
        "webflow": {
          "total": 19,
          "blocking": 0
        },
        "astro": {
          "total": 21,
          "blocking": 3
        },
        "framer": {
          "total": 10,
          "blocking": 0
        },
        "wix": {
          "total": 6,
          "blocking": 0
        },
        "typo3": {
          "total": 10,
          "blocking": 0
        },
        "drupal": {
          "total": 114,
          "blocking": 2
        },
        "sanity": {
          "total": 30,
          "blocking": 3
        },
        "shopify": {
          "total": 8,
          "blocking": 0
        },
        "salesforce-commerce": {
          "total": 1,
          "blocking": 0
        },
        "bigcommerce": {
          "total": 1,
          "blocking": 0
        },
        "storyblok": {
          "total": 5,
          "blocking": 0
        },
        "angular": {
          "total": 9,
          "blocking": 3
        },
        "ghost": {
          "total": 1,
          "blocking": 1
        },
        "remix": {
          "total": 3,
          "blocking": 1
        },
        "joomla": {
          "total": 4,
          "blocking": 0
        },
        "sveltekit": {
          "total": 4,
          "blocking": 2
        },
        "magento": {
          "total": 1,
          "blocking": 1
        },
        "squarespace": {
          "total": 1,
          "blocking": 0
        },
        "woocommerce": {
          "total": 2,
          "blocking": 0
        }
      },
      "perNetwork": {
        "cloudflare": {
          "total": 1082,
          "blocking": 187
        },
        "fastly": {
          "total": 216,
          "blocking": 53
        },
        "cloudfront": {
          "total": 413,
          "blocking": 82
        },
        "alibaba": {
          "total": 37,
          "blocking": 5
        },
        "azure-frontdoor": {
          "total": 28,
          "blocking": 1
        },
        "akamai": {
          "total": 265,
          "blocking": 43
        },
        "imperva": {
          "total": 11,
          "blocking": 1
        },
        "google": {
          "total": 153,
          "blocking": 5
        },
        "vercel": {
          "total": 48,
          "blocking": 3
        },
        "netlify": {
          "total": 25,
          "blocking": 1
        },
        "varnish": {
          "total": 39,
          "blocking": 12
        },
        "bunny": {
          "total": 11,
          "blocking": 0
        },
        "tencent": {
          "total": 1,
          "blocking": 0
        },
        "sucuri": {
          "total": 1,
          "blocking": 0
        }
      }
    }
  ],
  "bulkDownload": "https://crawlindex.org/data/domains.jsonl",
  "license": "CC BY 4.0",
  "attribution": "Fidget Labs BV, https://crawlindex.org"
}