Vextorium · robots.txt & AI crawler rules (x402)
https://api.vextorium.com/v1/robots-txtOverview
Parse a site's robots.txt: whether a user-agent may crawl a path, which AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, CCBot...) are blocked, and declared sitemaps. Check before your agent scrapes.
Protocol
x402 (POST)
Price
$0.0030
Payment Asset
0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913
Payment Network
eip155:8453
Category
uncategorized
Provider
Vextorium
Source
bazaar
Indexed
2026-10-03 21:23:32
Health
Status
healthy
Latency (p50)
144ms
Uptime (30d)
100.0%
Reliability Score
100/100
Last Checked
2026-10-31 07:03:44
Last Healthy
2026-10-31 07:03:44
Consecutive Failures
0
x402 Payment Validation
Payment Requirements
Valid
Asset Verified
Known USDC
Facilitator
—
Input Schema
{
"body": {
"agent": "GPTBot",
"route": "/section/world",
"url": "https://www.nytimes.com"
},
"bodyType": "json",
"method": "POST",
"type": "http"
}
Output Schema
{
"example": {
"ai_bots": {
"Amazonbot": "permitido",
"Applebot-Extended": "bloqueado",
"Bytespider": "bloqueado",
"CCBot": "bloqueado",
"ChatGPT-User": "bloqueado",
"Claude-SearchBot": "bloqueado",
"Claude-User": "bloqueado",
"ClaudeBot": "bloqueado",
"Diffbot": "bloqueado",
"DuckAssistBot": "bloqueado",
"GPTBot": "bloqueado",
"Google-Extended": "bloqueado",
"MistralAI-User": "permitido",
"OAI-SearchBot": "bloqueado",
"Perplexity-User": "bloqueado",
"PerplexityBot": "bloqueado",
"YouBot": "bloqueado",
"anthropic-ai": "bloqueado",
"cohere-ai": "bloqueado",
"meta-externalagent": "bloqueado"
},
"blocked_ai_bots": 18,
"groups": 63,
"query": {
"agent": "GPTBot",
"allowed": false,
"applied_group": [
"gptbot"
],
"route": "/section/world"
},
"robots_txt": true,
"site": "https://www.nytimes.com",
"sitemaps": [
"https://www.nytimes.com/sitemaps/new/news.xml.gz",
"https://www.nytimes.com/sitemaps/new/sitemap.xml.gz"
]
},
"type": "json"
}