{
  "benchmark": "CVE-Bench",
  "version": "v2.1.0",
  "lastUpdated": "2026-09-02",
  "metric": "pass@1",
  "scope": "40 live web-application CVE targets",
  "grader": "deterministic oracle",
  "license": "MIT",
  "source": "https://getfailsafe.com/cvebench",
  "leaderboard": true,
  "submissions": "Open for submissions through the official CVE-Bench project; rows retain their reported evaluation budget and comparability notes.",
  "results": [
    {
      "system": "FailSafe SWARM",
      "model": "DeepSeek-V4 + Kimi-K2.7",
      "zeroDay": { "solved": 25, "total": 40, "percentage": 62.5 },
      "oneDay": { "solved": 28, "total": 40, "percentage": 70.0 },
      "budget": "pass@1",
      "reported": "2026-08",
      "source": "https://github.com/failsafe-security/failsafe-swarm",
      "leaderboard": true
    },
    {
      "system": "CyberEvolver",
      "model": "Kimi-K2.5",
      "zeroDay": { "solved": 15, "total": 40, "percentage": 37.5 },
      "oneDay": { "solved": 18, "total": 40, "percentage": 45.0 },
      "budget": "16-node union",
      "reported": "2026-05",
      "source": "https://arxiv.org/abs/2605.26195",
      "leaderboard": true
    },
    {
      "system": "Claude Opus 4.6",
      "model": "Official CVE-Bench leaderboard entry",
      "zeroDay": { "solved": 13, "total": 40, "percentage": 32.5 },
      "oneDay": { "solved": 16, "total": 40, "percentage": 40.0 },
      "budget": "pass@1",
      "reported": "2026-04",
      "source": "https://cvebench.com/",
      "leaderboard": true
    },
    {
      "system": "CyberEvolver",
      "model": "Qwen3-235B",
      "zeroDay": { "solved": 13, "total": 40, "percentage": 32.5 },
      "oneDay": { "solved": 13, "total": 40, "percentage": 32.5 },
      "budget": "16-node union",
      "reported": "2026-05",
      "source": "https://arxiv.org/abs/2605.26195",
      "leaderboard": true
    },
    {
      "system": "CyberEvolver",
      "model": "MiniMax-M2.5",
      "zeroDay": { "solved": 11, "total": 40, "percentage": 27.5 },
      "oneDay": { "solved": 16, "total": 40, "percentage": 40.0 },
      "budget": "16-node union",
      "reported": "2026-05",
      "source": "https://arxiv.org/abs/2605.26195",
      "leaderboard": true
    },
    {
      "system": "CyberEvolver",
      "model": "DeepSeek-V3.1",
      "zeroDay": { "solved": 10, "total": 40, "percentage": 25.0 },
      "oneDay": { "solved": 13, "total": 40, "percentage": 32.5 },
      "budget": "16-node union",
      "reported": "2026-05",
      "source": "https://arxiv.org/abs/2605.26195",
      "leaderboard": true
    },
    {
      "system": "T-Agent",
      "model": "DeepSeek-V3.1",
      "zeroDay": { "solved": 7, "total": 40, "percentage": 17.5 },
      "oneDay": { "solved": 8, "total": 40, "percentage": 20.0 },
      "budget": "pass@4",
      "reported": "2026-05",
      "source": "https://arxiv.org/abs/2605.26195",
      "leaderboard": true
    },
    {
      "system": "CyAgent",
      "model": "DeepSeek-V3.1",
      "zeroDay": { "solved": 6, "total": 40, "percentage": 15.0 },
      "oneDay": { "solved": 5, "total": 40, "percentage": 12.5 },
      "budget": "pass@4",
      "reported": "2026-05",
      "source": "https://arxiv.org/abs/2605.26195",
      "leaderboard": true
    },
    {
      "system": "PenForge",
      "model": "Claude-3.7-Sonnet",
      "zeroDay": { "solved": null, "total": 40, "percentage": 20.5 },
      "oneDay": null,
      "budget": "pass@1",
      "reported": "2026-01",
      "source": "https://arxiv.org/abs/2601.06910",
      "leaderboard": false,
      "comparability": "Paper does not state a CVE-Bench version; listed for context, not ranked against v2.1.0."
    }
  ],
  "notes": [
    "Zero-day means no source code or vulnerability description is provided.",
    "One-day provides the NVD vulnerability description without source code.",
    "Autonomous runs are stochastic; the published repository contains the method and graded evidence."
  ]
}