robots.txt checker: whether GPTBot, ClaudeBot, CCBot and other AI crawlers may c...
https://minty-sprout.onrender.com/v1/robots-txtOverview
robots.txt checker: whether GPTBot, ClaudeBot, CCBot and other AI crawlers may crawl a site, plus sitemaps. Input: a domain or URL.
Protocol
x402 (POST)
Price
$0.0010
Payment Asset
USD Coin
Payment Network
eip155:8453
Category
uncategorized
Provider
Foundry
Source
self-registered
Indexed
2026-10-31 22:48:46
Health
Status
healthy
Latency (p50)
—
Uptime (30d)
—
Reliability Score
—
Last Checked
Never
Last Healthy
Never
Consecutive Failures
0
x402 Payment Validation
Payment Requirements
—
Asset Verified
—
Facilitator
—
Input Schema
{
"body": {
"input": "nytimes.com"
},
"bodyType": "json",
"method": "POST",
"type": "http"
}
Output Schema
{
"example": {
"ai_crawlers": {
"Applebot-Extended": false,
"Bytespider": false,
"CCBot": false,
"ChatGPT-User": false,
"Claude-User": false,
"ClaudeBot": false,
"GPTBot": false,
"Google-Extended": false,
"PerplexityBot": false,
"meta-externalagent": false
},
"crawl_delay": null,
"found": true,
"googlebot_allowed": true,
"host": "nytimes.com",
"lines": 364,
"path_checked": "/",
"sitemaps": [
"https://www.nytimes.com/sitemaps/new/news.xml.gz",
"https://www.nytimes.com/sitemaps/new/sitemap.xml.gz",
"https://www.nytimes.com/sitemaps/new/collections.xml.gz"
],
"status": 200
},
"type": "json"
}