Checks if AI crawlers can read up to 20 websites: GPTBot, OAI-SearchBot, ClaudeB...
https://x402-agent-tools.nerolabs.workers.dev/ai-crawlersOverview
Checks if AI crawlers can read up to 20 websites: GPTBot, OAI-SearchBot, ClaudeBot, PerplexityBot and Meta-ExternalAgent by default (four more optional). Reads robots.txt, fetches each site as each bot vs a browser to catch firewall blocks, and checks llms.txt. Returns allowed or blocked per bot and why. $0.005 per website. Unreachable sites are not charged. Runs nerolabs/ai-crawler-access-checker.
Protocol
x402 (POST)
Price
$0.01
Payment Asset
USD Coin
Payment Network
eip155:8453
Category
tools/search
Provider
Nero Labs Agent Tools
Source
bazaar
Indexed
2026-11-01 15:05:56
Health
Status
unknown
Latency (p50)
—
Uptime (30d)
—
Reliability Score
—
Last Checked
Never
Last Healthy
Never
Consecutive Failures
0
x402 Payment Validation
Payment Requirements
—
Asset Verified
—
Facilitator
—
Input Schema
{
"body": {
"websites": [
"nytimes.com",
"wikipedia.org"
]
},
"bodyType": "json",
"method": "POST",
"type": "http"
}
Output Schema
{
"example": {
"count": 1,
"rows": [
{
"allowedBots": [
"GPTBot",
"OAI-SearchBot",
"PerplexityBot"
],
"blockedBots": [
"ClaudeBot",
"Meta-ExternalAgent"
],
"blockedCount": 2,
"blocksAiSearch": false,
"blocksTraining": true,
"bots": [
{
"blocked": false,
"blockedBy": null,
"bot": "GPTBot",
"disallowRules": 428,
"fetchHttpStatus": 200,
"purpose": "training",
"realFetch": "served",
"robotsTxt": "allowed",
"ruleSource": "wildcard",
"vendor": "OpenAI"
}
],
"botsChecked": 5,
"changes": [],
"checkStatus": "ok",
"checkedAt": "2026-11-01T14:47:36.934Z",
"checkedUrl": "https://wikipedia.org",
"claudebotFetch": "blocked",
"claudebotRobots": "allowed",
"duplicateOf": null,
"edge": null,
"finalUrl": "https://www.wikipedia.org/",
"fixes": [
"ClaudeBot gets HTTP 403 from your server or firewall although robots.txt allows it. Anthropic will not use this site to train Claude. Check your CDN or bot-protection rules."
],
"gptbotFetch": "served",
"gptbotRobots": "allowed",
"homepageHttpStatus": 200,
"llmsFullTxtFound": false,
"llmsTxtFound": false,
"llmsTxtTitle": null,
"llmsTxtUrl": null,
"metaExternalagentFetch": "blocked",
"metaExternalagentRobots": "allowed",
"monitorStatus": null,
"noaiDirective": false,
"oaiSearchbotFetch": "served",
"oaiSearchbotRobots": "allowed",
"perplexitybotFetch": "served",
"perplexitybotRobots": "allowed",
"previousCheckedAt": null,
"realFetchTested": true,
"robotsTxtStatus": "found",
"robotsTxtUrl": "https://www.wikipedia.org/robots.txt",
"rowNumber": 2,
"statusDetail": null,
"summary": "Blocks ClaudeBot, Meta-ExternalAgent; allows GPTBot, OAI-SearchBot, PerplexityBot. No llms.txt.",
"verdict": "partly_blocked",
"viaProxy": false
}
]
},
"type": "json"
}