{
  "title": "Agentic pentest buyer evaluation checklist",
  "publisher": "FailSafe",
  "updatedAt": "2026-11-02",
  "source": "https://getfailsafe.com/agentic-pentesting#evaluation",
  "method": "First-party buyer framework, not a certification or vendor ranking. Compare providers on the same authorized target and access. Record observed evidence, limits and unanswered questions.",
  "checks": [
    { "area": "Authorization", "ask": "Record targets, exclusions, allowed actions, rate limits, emergency contact and tested stop controls." },
    { "area": "Coverage", "ask": "Specify assets, user roles, access, business workflows and untested surfaces." },
    { "area": "Evidence", "ask": "Request sanitized preconditions, reproduction steps, observed impact and reviewer decision." },
    { "area": "Remediation", "ask": "Demonstrate ticket handoff, permissions for proposed code changes, engineering approval and a retest tied to the changed build." },
    { "area": "Inference", "ask": "Document the location of execution, model serving, logs, evidence and fallback routes. Verify egress rather than inferring locality from a runner." },
    { "area": "Model data", "ask": "Confirm retention and deletion. Customer data is not used for FailSafe model training by default; explicit permission is required." },
    { "area": "Benchmarks", "ask": "Compare version, sample, access, attempt budget, grading and public artifacts. Keep customer outcomes separate." }
  ]
}
