{
  "title": "DistDebug-Bench leaderboard",
  "generated": "2026-11-02",
  "source": {
    "paper_commit": "b922b0b",
    "file": "data/eval_summary.json",
    "sha256": "6093738c9034870f90e94e23c76f04e42eb247824ae975dadbc08cec13b41f57",
    "uncommitted_changes": false,
    "traces": "https://huggingface.co/datasets/OrderLab/distdebug-bench-traces"
  },
  "protocol": "120 incidents x 3 attempts; the first three schema-valid attempts per incident are scored; block coverage is the primary metric, with incident-bootstrap 95% CIs.",
  "effort": "Every configuration runs at medium reasoning effort.",
  "weights_checked": "2026-10-30",
  "units": "Coverage, precision, and flaky rates are percentages; tokens per run are thousands of input plus output tokens. calls_per_run is null where the harness does not report it.",
  "default_sort": "block_coverage",
  "evidence_groups": [
    {
      "key": "method_no_stack",
      "report": "Causal method named",
      "stack": "No internal stack trace",
      "incidents": 16
    },
    {
      "key": "opaque_no_stack",
      "report": "No causal code named",
      "stack": "No internal stack trace",
      "incidents": 42
    },
    {
      "key": "opaque_cross_component",
      "report": "No causal code named",
      "stack": "Cross-component crash site",
      "incidents": 7
    }
  ],
  "evidence_units": "In each report-evidence group: block coverage and its 95% incident-bootstrap CI, in percent; within_sd, the mean over incidents of the standard deviation of block coverage across attempts, in percentage points; flaky, the number of incidents whose block coverage spans more than 20 points across attempts.",
  "rows": [
    {
      "id": "mini-swe-agent-gpt-6-astra",
      "paper_order": 1,
      "model": "GPT-6 Astra",
      "chart_label": "GPT-6 Astra",
      "harness": "Mini-SWE-Agent",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-gpt-6-astra-medium-effort",
      "values": {
        "file_coverage": 83.9,
        "function_coverage": 58.4,
        "line_coverage": 53.9,
        "block_coverage": 67.7,
        "file_precision": 66.4,
        "function_precision": 37.8,
        "line_precision": 19.9,
        "block_precision": 42.7,
        "block_coverage_ci": [
          60.4,
          74.9
        ],
        "flaky_bugs": 15,
        "flaky_rate": 12,
        "blocks_per_diagnosis": 3.3,
        "calls_per_run": 9.0,
        "tokens_per_run_k": 132
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 94.8,
          "block_coverage_ci": [
            86.5,
            100.0
          ],
          "within_sd": 1.5,
          "flaky": 1
        },
        "opaque_no_stack": {
          "block_coverage": 65.0,
          "block_coverage_ci": [
            52.8,
            76.7
          ],
          "within_sd": 6.4,
          "flaky": 7
        },
        "opaque_cross_component": {
          "block_coverage": 38.1,
          "block_coverage_ci": [
            9.5,
            71.4
          ],
          "within_sd": 10.1,
          "flaky": 2
        }
      }
    },
    {
      "id": "mini-swe-agent-claude-opus-5-5",
      "paper_order": 2,
      "model": "Claude Opus 5.5",
      "chart_label": "Claude Opus 5.5",
      "harness": "Mini-SWE-Agent",
      "family": "anthropic",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-claude-opus-5.5-medium-effort",
      "values": {
        "file_coverage": 82.0,
        "function_coverage": 60.4,
        "line_coverage": 53.5,
        "block_coverage": 66.4,
        "file_precision": 67.4,
        "function_precision": 39.2,
        "line_precision": 21.0,
        "block_precision": 45.5,
        "block_coverage_ci": [
          59.2,
          73.6
        ],
        "flaky_bugs": 17,
        "flaky_rate": 14,
        "blocks_per_diagnosis": 2.8,
        "calls_per_run": 9.2,
        "tokens_per_run_k": 193
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 91.7,
          "block_coverage_ci": [
            82.3,
            100.0
          ],
          "within_sd": 1.3,
          "flaky": 1
        },
        "opaque_no_stack": {
          "block_coverage": 62.0,
          "block_coverage_ci": [
            49.8,
            73.6
          ],
          "within_sd": 6.8,
          "flaky": 8
        },
        "opaque_cross_component": {
          "block_coverage": 40.5,
          "block_coverage_ci": [
            11.9,
            71.4
          ],
          "within_sd": 6.7,
          "flaky": 1
        }
      }
    },
    {
      "id": "mini-swe-agent-gpt-5-5",
      "paper_order": 3,
      "model": "GPT-5.5",
      "chart_label": "GPT-5.5",
      "harness": "Mini-SWE-Agent",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-gpt-5.5-medium-effort-fixed",
      "values": {
        "file_coverage": 84.0,
        "function_coverage": 55.5,
        "line_coverage": 52.7,
        "block_coverage": 65.1,
        "file_precision": 58.1,
        "function_precision": 29.1,
        "line_precision": 14.6,
        "block_precision": 34.7,
        "block_coverage_ci": [
          57.8,
          72.3
        ],
        "flaky_bugs": 20,
        "flaky_rate": 17,
        "blocks_per_diagnosis": 3.6,
        "calls_per_run": 21.9,
        "tokens_per_run_k": 468
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 88.5,
          "block_coverage_ci": [
            76.0,
            100.0
          ],
          "within_sd": 3.7,
          "flaky": 2
        },
        "opaque_no_stack": {
          "block_coverage": 60.1,
          "block_coverage_ci": [
            48.4,
            71.6
          ],
          "within_sd": 9.2,
          "flaky": 11
        },
        "opaque_cross_component": {
          "block_coverage": 28.6,
          "block_coverage_ci": [
            0.0,
            57.1
          ],
          "within_sd": 0.0,
          "flaky": 0
        }
      }
    },
    {
      "id": "mini-swe-agent-gpt-5-4",
      "paper_order": 4,
      "model": "GPT-5.4",
      "chart_label": "GPT-5.4",
      "harness": "Mini-SWE-Agent",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-gpt-5.4-medium-effort-combined",
      "values": {
        "file_coverage": 79.8,
        "function_coverage": 53.1,
        "line_coverage": 50.4,
        "block_coverage": 63.1,
        "file_precision": 59.9,
        "function_precision": 32.3,
        "line_precision": 14.6,
        "block_precision": 39.5,
        "block_coverage_ci": [
          56.0,
          70.2
        ],
        "flaky_bugs": 21,
        "flaky_rate": 18,
        "blocks_per_diagnosis": 2.9,
        "calls_per_run": 25.0,
        "tokens_per_run_k": 690
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 92.7,
          "block_coverage_ci": [
            81.2,
            100.0
          ],
          "within_sd": 0.7,
          "flaky": 1
        },
        "opaque_no_stack": {
          "block_coverage": 56.4,
          "block_coverage_ci": [
            45.2,
            67.3
          ],
          "within_sd": 11.1,
          "flaky": 12
        },
        "opaque_cross_component": {
          "block_coverage": 16.7,
          "block_coverage_ci": [
            0.0,
            40.5
          ],
          "within_sd": 10.1,
          "flaky": 2
        }
      }
    },
    {
      "id": "mini-swe-agent-gpt-5-3-codex",
      "paper_order": 5,
      "model": "GPT-5.3-Codex",
      "chart_label": "5.3-Codex",
      "harness": "Mini-SWE-Agent",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-gpt-5.3-codex-medium-effort-combined",
      "values": {
        "file_coverage": 79.1,
        "function_coverage": 53.0,
        "line_coverage": 44.5,
        "block_coverage": 59.7,
        "file_precision": 63.2,
        "function_precision": 35.2,
        "line_precision": 17.8,
        "block_precision": 40.0,
        "block_coverage_ci": [
          52.2,
          67.2
        ],
        "flaky_bugs": 22,
        "flaky_rate": 18,
        "blocks_per_diagnosis": 2.6,
        "calls_per_run": 22.7,
        "tokens_per_run_k": 416
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 87.2,
          "block_coverage_ci": [
            76.4,
            95.8
          ],
          "within_sd": 7.6,
          "flaky": 5
        },
        "opaque_no_stack": {
          "block_coverage": 50.4,
          "block_coverage_ci": [
            37.9,
            62.5
          ],
          "within_sd": 6.5,
          "flaky": 8
        },
        "opaque_cross_component": {
          "block_coverage": 16.7,
          "block_coverage_ci": [
            0.0,
            40.5
          ],
          "within_sd": 10.1,
          "flaky": 2
        }
      }
    },
    {
      "id": "mini-swe-agent-mimo-v2-5-pro",
      "paper_order": 6,
      "model": "MiMo v2.5 Pro",
      "chart_label": "MiMo v2.5 Pro",
      "harness": "Mini-SWE-Agent",
      "family": "mimo",
      "weights": "open",
      "weights_license": "MIT",
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-mimo-v2.5-pro",
      "values": {
        "file_coverage": 78.9,
        "function_coverage": 60.1,
        "line_coverage": 43.4,
        "block_coverage": 58.5,
        "file_precision": 62.3,
        "function_precision": 37.8,
        "line_precision": 17.3,
        "block_precision": 38.1,
        "block_coverage_ci": [
          50.8,
          66.1
        ],
        "flaky_bugs": 22,
        "flaky_rate": 18,
        "blocks_per_diagnosis": 2.9,
        "calls_per_run": 55.5,
        "tokens_per_run_k": 2643
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 86.3,
          "block_coverage_ci": [
            75.7,
            95.3
          ],
          "within_sd": 4.7,
          "flaky": 3
        },
        "opaque_no_stack": {
          "block_coverage": 49.5,
          "block_coverage_ci": [
            37.4,
            61.6
          ],
          "within_sd": 8.4,
          "flaky": 11
        },
        "opaque_cross_component": {
          "block_coverage": 28.6,
          "block_coverage_ci": [
            0.0,
            57.1
          ],
          "within_sd": 0.0,
          "flaky": 0
        }
      }
    },
    {
      "id": "mini-swe-agent-kimi-k2-6",
      "paper_order": 7,
      "model": "Kimi K2.6",
      "chart_label": "Kimi K2.6",
      "harness": "Mini-SWE-Agent",
      "family": "kimi",
      "weights": "open",
      "weights_license": "Modified MIT",
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-kimi-k2.6",
      "values": {
        "file_coverage": 77.5,
        "function_coverage": 51.2,
        "line_coverage": 44.8,
        "block_coverage": 58.3,
        "file_precision": 66.3,
        "function_precision": 38.8,
        "line_precision": 19.6,
        "block_precision": 45.9,
        "block_coverage_ci": [
          50.9,
          65.6
        ],
        "flaky_bugs": 32,
        "flaky_rate": 27,
        "blocks_per_diagnosis": 2.4,
        "calls_per_run": 66.1,
        "tokens_per_run_k": 2796
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 91.3,
          "block_coverage_ci": [
            83.0,
            97.9
          ],
          "within_sd": 3.2,
          "flaky": 3
        },
        "opaque_no_stack": {
          "block_coverage": 49.7,
          "block_coverage_ci": [
            38.6,
            60.9
          ],
          "within_sd": 14.4,
          "flaky": 15
        },
        "opaque_cross_component": {
          "block_coverage": 16.7,
          "block_coverage_ci": [
            0.0,
            40.5
          ],
          "within_sd": 10.1,
          "flaky": 2
        }
      }
    },
    {
      "id": "mini-swe-agent-mimo-v2-5",
      "paper_order": 8,
      "model": "MiMo v2.5",
      "chart_label": "MiMo v2.5",
      "harness": "Mini-SWE-Agent",
      "family": "mimo",
      "weights": "open",
      "weights_license": "MIT",
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-mimo-v2.5",
      "values": {
        "file_coverage": 77.8,
        "function_coverage": 54.2,
        "line_coverage": 43.9,
        "block_coverage": 58.0,
        "file_precision": 59.5,
        "function_precision": 33.4,
        "line_precision": 16.4,
        "block_precision": 36.5,
        "block_coverage_ci": [
          50.5,
          65.4
        ],
        "flaky_bugs": 26,
        "flaky_rate": 22,
        "blocks_per_diagnosis": 3.1,
        "calls_per_run": 49.7,
        "tokens_per_run_k": 2332
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 84.5,
          "block_coverage_ci": [
            70.1,
            96.0
          ],
          "within_sd": 3.2,
          "flaky": 3
        },
        "opaque_no_stack": {
          "block_coverage": 47.3,
          "block_coverage_ci": [
            35.3,
            59.4
          ],
          "within_sd": 9.1,
          "flaky": 12
        },
        "opaque_cross_component": {
          "block_coverage": 23.8,
          "block_coverage_ci": [
            0.0,
            52.4
          ],
          "within_sd": 3.4,
          "flaky": 1
        }
      }
    },
    {
      "id": "mini-swe-agent-deepseek-v4-flash",
      "paper_order": 9,
      "model": "DeepSeek V4 Flash",
      "chart_label": "DeepSeek V4 Flash",
      "harness": "Mini-SWE-Agent",
      "family": "deepseek",
      "weights": "open",
      "weights_license": "MIT",
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-deepseek-v4-flash-azure",
      "values": {
        "file_coverage": 76.0,
        "function_coverage": 49.4,
        "line_coverage": 45.9,
        "block_coverage": 56.4,
        "file_precision": 57.4,
        "function_precision": 28.6,
        "line_precision": 13.0,
        "block_precision": 32.7,
        "block_coverage_ci": [
          49.0,
          63.7
        ],
        "flaky_bugs": 32,
        "flaky_rate": 27,
        "blocks_per_diagnosis": 3.2,
        "calls_per_run": 61.6,
        "tokens_per_run_k": 2291
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 83.3,
          "block_coverage_ci": [
            72.9,
            92.7
          ],
          "within_sd": 9.1,
          "flaky": 4
        },
        "opaque_no_stack": {
          "block_coverage": 45.4,
          "block_coverage_ci": [
            34.1,
            56.9
          ],
          "within_sd": 12.8,
          "flaky": 15
        },
        "opaque_cross_component": {
          "block_coverage": 16.7,
          "block_coverage_ci": [
            0.0,
            40.5
          ],
          "within_sd": 10.1,
          "flaky": 2
        }
      }
    },
    {
      "id": "mini-swe-agent-gpt-5-2-codex",
      "paper_order": 10,
      "model": "GPT-5.2-Codex",
      "chart_label": "5.2-Codex",
      "harness": "Mini-SWE-Agent",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-gpt-5.2-codex",
      "values": {
        "file_coverage": 75.9,
        "function_coverage": 48.8,
        "line_coverage": 45.9,
        "block_coverage": 56.4,
        "file_precision": 63.8,
        "function_precision": 35.4,
        "line_precision": 16.1,
        "block_precision": 43.1,
        "block_coverage_ci": [
          48.5,
          64.2
        ],
        "flaky_bugs": 18,
        "flaky_rate": 15,
        "blocks_per_diagnosis": 2.3,
        "calls_per_run": 32.1,
        "tokens_per_run_k": 666
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 82.8,
          "block_coverage_ci": [
            67.2,
            95.8
          ],
          "within_sd": 1.5,
          "flaky": 1
        },
        "opaque_no_stack": {
          "block_coverage": 44.7,
          "block_coverage_ci": [
            32.2,
            57.4
          ],
          "within_sd": 7.7,
          "flaky": 11
        },
        "opaque_cross_component": {
          "block_coverage": 14.3,
          "block_coverage_ci": [
            0.0,
            42.9
          ],
          "within_sd": 0.0,
          "flaky": 0
        }
      }
    },
    {
      "id": "mini-swe-agent-hy3",
      "paper_order": 11,
      "model": "HY3",
      "chart_label": "HY3",
      "harness": "Mini-SWE-Agent",
      "family": "hunyuan",
      "weights": "open",
      "weights_license": "Tencent Hy Community License",
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-hy3",
      "values": {
        "file_coverage": 72.5,
        "function_coverage": 49.1,
        "line_coverage": 39.7,
        "block_coverage": 51.6,
        "file_precision": 60.9,
        "function_precision": 35.6,
        "line_precision": 15.5,
        "block_precision": 38.7,
        "block_coverage_ci": [
          44.1,
          59.1
        ],
        "flaky_bugs": 35,
        "flaky_rate": 29,
        "blocks_per_diagnosis": 2.7,
        "calls_per_run": 32.5,
        "tokens_per_run_k": 768
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 78.8,
          "block_coverage_ci": [
            64.2,
            91.7
          ],
          "within_sd": 6.9,
          "flaky": 4
        },
        "opaque_no_stack": {
          "block_coverage": 40.2,
          "block_coverage_ci": [
            28.4,
            52.3
          ],
          "within_sd": 10.5,
          "flaky": 13
        },
        "opaque_cross_component": {
          "block_coverage": 11.9,
          "block_coverage_ci": [
            0.0,
            26.2
          ],
          "within_sd": 12.6,
          "flaky": 2
        }
      }
    },
    {
      "id": "mini-swe-agent-gemini-3-5-flash",
      "paper_order": 12,
      "model": "Gemini 3.5 Flash",
      "chart_label": "Gemini 3.5",
      "harness": "Mini-SWE-Agent",
      "family": "gemini",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-v2-gemini-3.5-flash",
      "values": {
        "file_coverage": 65.9,
        "function_coverage": 43.5,
        "line_coverage": 37.3,
        "block_coverage": 47.5,
        "file_precision": 68.0,
        "function_precision": 45.4,
        "line_precision": 23.0,
        "block_precision": 51.1,
        "block_coverage_ci": [
          39.7,
          55.2
        ],
        "flaky_bugs": 22,
        "flaky_rate": 18,
        "blocks_per_diagnosis": 1.4,
        "calls_per_run": 25.0,
        "tokens_per_run_k": 498
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 77.1,
          "block_coverage_ci": [
            60.4,
            90.6
          ],
          "within_sd": 0.0,
          "flaky": 0
        },
        "opaque_no_stack": {
          "block_coverage": 38.5,
          "block_coverage_ci": [
            26.3,
            51.0
          ],
          "within_sd": 7.6,
          "flaky": 9
        },
        "opaque_cross_component": {
          "block_coverage": 9.5,
          "block_coverage_ci": [
            0.0,
            28.6
          ],
          "within_sd": 3.4,
          "flaky": 1
        }
      }
    },
    {
      "id": "mini-swe-agent-o4-mini",
      "paper_order": 13,
      "model": "o4-mini",
      "chart_label": "o4-mini",
      "harness": "Mini-SWE-Agent",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "mini-swe-agent-o4-mini",
      "values": {
        "file_coverage": 63.1,
        "function_coverage": 35.4,
        "line_coverage": 28.2,
        "block_coverage": 39.3,
        "file_precision": 60.5,
        "function_precision": 32.1,
        "line_precision": 16.7,
        "block_precision": 34.7,
        "block_coverage_ci": [
          32.1,
          46.8
        ],
        "flaky_bugs": 30,
        "flaky_rate": 25,
        "blocks_per_diagnosis": 1.8,
        "calls_per_run": 29.9,
        "tokens_per_run_k": 498
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 66.3,
          "block_coverage_ci": [
            47.6,
            84.0
          ],
          "within_sd": 11.6,
          "flaky": 6
        },
        "opaque_no_stack": {
          "block_coverage": 24.3,
          "block_coverage_ci": [
            14.5,
            34.8
          ],
          "within_sd": 8.1,
          "flaky": 9
        },
        "opaque_cross_component": {
          "block_coverage": 0.0,
          "block_coverage_ci": [
            0.0,
            0.0
          ],
          "within_sd": 0.0,
          "flaky": 0
        }
      }
    },
    {
      "id": "codex-cli-gpt-5-2-codex",
      "paper_order": 14,
      "model": "GPT-5.2-Codex",
      "chart_label": "5.2-Codex",
      "harness": "Codex CLI",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "codex_5.2_codex_medium_effort_mar30",
      "values": {
        "file_coverage": 64.6,
        "function_coverage": 46.8,
        "line_coverage": 35.5,
        "block_coverage": 46.5,
        "file_precision": 66.9,
        "function_precision": 47.6,
        "line_precision": 24.1,
        "block_precision": 49.3,
        "block_coverage_ci": [
          38.6,
          54.5
        ],
        "flaky_bugs": 17,
        "flaky_rate": 14,
        "blocks_per_diagnosis": 1.6,
        "calls_per_run": null,
        "tokens_per_run_k": 500
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 67.2,
          "block_coverage_ci": [
            47.4,
            85.9
          ],
          "within_sd": 1.5,
          "flaky": 1
        },
        "opaque_no_stack": {
          "block_coverage": 35.3,
          "block_coverage_ci": [
            23.9,
            47.3
          ],
          "within_sd": 7.5,
          "flaky": 7
        },
        "opaque_cross_component": {
          "block_coverage": 9.5,
          "block_coverage_ci": [
            0.0,
            28.6
          ],
          "within_sd": 3.4,
          "flaky": 1
        }
      }
    },
    {
      "id": "codex-cli-o4-mini",
      "paper_order": 15,
      "model": "o4-mini",
      "chart_label": "o4-mini",
      "harness": "Codex CLI",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "codex_o4_mini",
      "values": {
        "file_coverage": 55.5,
        "function_coverage": 36.7,
        "line_coverage": 23.6,
        "block_coverage": 34.4,
        "file_precision": 56.1,
        "function_precision": 36.2,
        "line_precision": 18.1,
        "block_precision": 33.4,
        "block_coverage_ci": [
          27.3,
          41.9
        ],
        "flaky_bugs": 27,
        "flaky_rate": 22,
        "blocks_per_diagnosis": 1.7,
        "calls_per_run": null,
        "tokens_per_run_k": 331
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 47.4,
          "block_coverage_ci": [
            26.0,
            69.3
          ],
          "within_sd": 4.4,
          "flaky": 3
        },
        "opaque_no_stack": {
          "block_coverage": 24.2,
          "block_coverage_ci": [
            14.9,
            34.6
          ],
          "within_sd": 11.6,
          "flaky": 12
        },
        "opaque_cross_component": {
          "block_coverage": 0.0,
          "block_coverage_ci": [
            0.0,
            0.0
          ],
          "within_sd": 0.0,
          "flaky": 0
        }
      }
    },
    {
      "id": "opencode-gpt-5-2-codex",
      "paper_order": 16,
      "model": "GPT-5.2-Codex",
      "chart_label": "5.2-Codex",
      "harness": "OpenCode",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "opencode-gpt-5.2-codex",
      "values": {
        "file_coverage": 69.1,
        "function_coverage": 45.3,
        "line_coverage": 37.1,
        "block_coverage": 50.0,
        "file_precision": 66.8,
        "function_precision": 42.8,
        "line_precision": 21.4,
        "block_precision": 47.7,
        "block_coverage_ci": [
          42.4,
          57.9
        ],
        "flaky_bugs": 23,
        "flaky_rate": 19,
        "blocks_per_diagnosis": 1.8,
        "calls_per_run": 18.2,
        "tokens_per_run_k": 532
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 70.3,
          "block_coverage_ci": [
            51.6,
            87.5
          ],
          "within_sd": 1.5,
          "flaky": 1
        },
        "opaque_no_stack": {
          "block_coverage": 39.8,
          "block_coverage_ci": [
            27.6,
            52.0
          ],
          "within_sd": 5.5,
          "flaky": 7
        },
        "opaque_cross_component": {
          "block_coverage": 14.3,
          "block_coverage_ci": [
            0.0,
            33.3
          ],
          "within_sd": 10.1,
          "flaky": 2
        }
      }
    },
    {
      "id": "opencode-o4-mini",
      "paper_order": 17,
      "model": "o4-mini",
      "chart_label": "o4-mini",
      "harness": "OpenCode",
      "family": "openai",
      "weights": "proprietary",
      "weights_license": null,
      "reasoning_effort": "medium",
      "attempts_per_bug": 3,
      "bugs": 120,
      "source_experiment": "opencode-o4-mini",
      "values": {
        "file_coverage": 54.2,
        "function_coverage": 30.5,
        "line_coverage": 23.5,
        "block_coverage": 33.5,
        "file_precision": 58.1,
        "function_precision": 34.5,
        "line_precision": 20.6,
        "block_precision": 38.4,
        "block_coverage_ci": [
          26.6,
          40.7
        ],
        "flaky_bugs": 25,
        "flaky_rate": 21,
        "blocks_per_diagnosis": 1.4,
        "calls_per_run": 13.5,
        "tokens_per_run_k": 280
      },
      "evidence": {
        "method_no_stack": {
          "block_coverage": 59.9,
          "block_coverage_ci": [
            40.1,
            79.2
          ],
          "within_sd": 5.9,
          "flaky": 3
        },
        "opaque_no_stack": {
          "block_coverage": 15.6,
          "block_coverage_ci": [
            8.5,
            23.3
          ],
          "within_sd": 6.6,
          "flaky": 6
        },
        "opaque_cross_component": {
          "block_coverage": 4.8,
          "block_coverage_ci": [
            0.0,
            14.3
          ],
          "within_sd": 3.4,
          "flaky": 1
        }
      }
    }
  ]
}