{
 "id": "robots-txt-check",
 "name": "robots.txt Rule Check (RFC 9309)",
 "state": "testing",
 "line": "Say whether a crawler may fetch a path under a robots.txt text, by RFC 9309: allowed or disallowed, the deciding rule and its line, the group used, and every line the protocol does not accept, with a signed record.",
 "url": "https://llpro.ai/robots-txt-check",
 "markdown": "https://llpro.ai/robots-txt-check.md",
 "division": "agents",
 "category": "tools",
 "type": "product",
 "breadcrumb": [
  {
   "name": "Home",
   "url": "https://llpro.ai/"
  },
  {
   "name": "Agents",
   "url": "https://llpro.ai/agents"
  },
  {
   "name": "Tools",
   "url": "https://llpro.ai/agents/tools"
  }
 ],
 "llms_txt": "https://llpro.ai/llms.txt",
 "mcp_server": "https://llpro.ai/mcp"
}