Agent-Ready Audit plus generated files: llms.txt, llms-full.txt, merged robots.t...

{ "body": { "crawler_policy": "allow_search_block_training", "max_pages": 5, "url": "https://www.python.org/" }, "bodyType": "json", "method": "POST", "type": "http" }

{ "example": { "artifacts": { "llms_txt": "# Python.org\n\n> The official home of the Python Programming Language\n\n## Pages\n\n- [Welcome to Python.org](https://www.python.org/): The official home of the Python Programming Language\n- [Python Software Foundation](https://www.python.org/psf-landing/): The official home of the Python Programming Language\n- [Python Job Board | Python.org](https://www.python.org/jobs/): The official home of the Python Programming Language\n- [Our Community | Python.org](https://www.python.org/community/): The offi", "robots_txt_ai_block": "# BEGIN agent-ready AI crawler rules (generated)\n# policy: allow_search_block_training (bot tokens from operator docs, accessed 2026-10-16)\n# allowed bots repeat your User-agent: * rules, because a named group replaces the * group (RFC 9309)\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /~guido/orlijn/\nDisallow: /webstats/\n\nUser-agent: Claude-S" }, "bots": [ { "class": "training", "http_probe": { "blocked_by_waf": false, "probed_as": "GPTBot", "status": 200 }, "matched_rule": "group * has no rule for / (allowed)", "name": "GPTBot", "operator": "OpenAI", "robots": "not_specified" }, { "class": "training", "http_probe": { "blocked_by_waf": false, "probed_as": "GPTBot", "status": 200 }, "matched_rule": "group * has no rule for / (allowed)", "name": "ClaudeBot", "operator": "Anthropic", "robots": "not_specified" }, { "class": "training", "http_probe": { "blocked_by_waf": false, "probed_as": "GPTBot", "status": 200 }, "matched_rule": "group * has no rule for / (allowed)", "name": "Google-Extended", "operator": "Google", "robots": "not_specified" } ], "bundle": { "bytes": 32631, "encoding": "base64", "filename": "agent-ready-kit.zip", "files": [ ".well-known/agent-card.json", "INSTALL.md", "jsonld/community.json", "jsonld/community_irc.json", "jsonld/index.json", "jsonld/jobs.json", "jsonld/psf-landing.json", "llms-full.txt", "llms.txt", "meta/community.html", "meta/community_irc.html", "meta/index.html", "meta/jobs.html", "meta/psf-landing.html", "robots.txt", "sitemap.xml" ] }, "checks": [ { "category": "machine_readable_summary", "evidence": "no /llms.txt", "fix": { "effort": "S", "snippet": "", "snippet_lang": "txt", "summary": "Publish /llms.txt in the llmstxt.org shape (see artifacts.llms_txt)." }, "id": "llms_txt", "max": 8, "pass": false, "points": 0, "severity": "medium" }, { "category": "machine_readable_summary", "evidence": "no /llms-full.txt", "fix": { "effort": "S", "snippet": "", "snippet_lang": "txt", "summary": "Publish /llms-full.txt with the full text of your key pages (see artifacts.llms_full_txt)." }, "id": "llms_full_txt", "max": 3, "pass": false, "points": 0, "severity": "low" } ], "disclaimer": "Improves technical accessibility for AI crawlers and agents. Does not guarantee crawling, ranking, citation, or traffic.", "grade": "D", "pages_audited": [ "https://www.python.org/", "https://www.python.org/psf-landing/", "https://www.python.org/jobs/" ], "reason": null, "score": 62.8, "skill": "agent_ready_kit", "subscores": { "agent_interfaces": null, "content_readability": 100, "crawler_access": 100, "discoverability": 43, "machine_readable_summary": 0, "structured_data": 50 }, "url": "https://www.python.org/" }, "type": "json" }