{
  "provenance": {
    "checked": "2026-10-30",
    "paper_root": "dsd-bench-neurips-ed-2026",
    "main_results": "figure/eval_main_results.tex",
    "intervals": "figure/eval_confidence_intervals.tex",
    "evidence_breakdown": "data/eval_summary.json: stratified.cells[no_stack_trace|method, no_stack_trace|opaque, cross_component|opaque] for mini_swe_agent_v2_gpt_6_astra_medium_effort; section/eval.tex:113-115",
    "flakiness": "section/eval.tex:130-148",
    "interval_method": "95% bug-bootstrap percentile intervals; 10,000 bug-level resamples, PCG64 seed 2026. Incident-sampling uncertainty, not rerun variability.",
    "status": "Complete: three scored attempts per incident for all 17 configurations (6,120 attempts).",
    "precision": "Displayed values and intervals rounded to one decimal as in the paper."
  },
  "consistency": {
    "model": "GPT-5.5",
    "harness": "Mini-SWE-Agent",
    "incidents": 120,
    "flaky": 20,
    "definition": "Within-incident block-coverage spread strictly greater than 20 percentage points across recorded attempts.",
    "groups": [
      {
        "label": "Causal method named",
        "flaky": 2,
        "incidents": 16,
        "mean_within_incident_sd_pp": 3.7
      },
      {
        "label": "No causal code named",
        "flaky": 11,
        "incidents": 42,
        "mean_within_incident_sd_pp": 9.2
      }
    ],
    "scope": "Both groups lack a usable internal stack trace; distinct incident groups.",
    "sd_ratio": 2.5,
    "sd_source": "data/eval_summary.json: stratified.cells[no_stack_trace|method / no_stack_trace|opaque].mean_within_bug_std[mini_swe_agent_v2_gpt_5_5_medium_effort_fixed]; section/eval.tex:140-144",
    "interpretation": "Observed differences between incident groups. Non-flaky does not mean a correct diagnosis; these measurements concern localization, not patches."
  },
  "models": {
    "gpt52": {
      "name": "GPT-5.2-Codex",
      "harness": "Mini-SWE-Agent",
      "coverage": 56.4,
      "coverage_ci": [
        48.5,
        64.2
      ],
      "precision": 43.1,
      "precision_ci": [
        36.6,
        49.5
      ],
      "tokens": 666000,
      "blocks": 2.3
    },
    "gpt55": {
      "name": "GPT-5.5",
      "harness": "Mini-SWE-Agent",
      "coverage": 65.1,
      "coverage_ci": [
        57.8,
        72.3
      ],
      "precision": 34.7,
      "precision_ci": [
        29.6,
        39.7
      ],
      "tokens": 468000,
      "blocks": 3.6
    },
    "deepseek": {
      "name": "DeepSeek V4 Flash",
      "harness": "Mini-SWE-Agent",
      "coverage": 56.4,
      "coverage_ci": [
        49.0,
        63.7
      ],
      "precision": 32.7,
      "precision_ci": [
        27.7,
        37.9
      ],
      "tokens": 2291000
    },
    "gpt6_astra": {
      "name": "GPT-6 Astra",
      "harness": "Mini-SWE-Agent",
      "coverage": 67.7,
      "coverage_ci": [
        60.4,
        74.9
      ],
      "precision": 42.7,
      "precision_ci": [
        36.6,
        48.9
      ],
      "tokens": 132000,
      "blocks": 3.3
    },
    "claude_opus_5_5": {
      "name": "Claude Opus 5.5",
      "harness": "Mini-SWE-Agent",
      "coverage": 66.4,
      "coverage_ci": [
        59.2,
        73.6
      ],
      "precision": 45.5,
      "precision_ci": [
        39.7,
        51.5
      ],
      "tokens": 193000,
      "blocks": 2.8
    },
    "gemini_3_5_flash": {
      "name": "Gemini 3.5 Flash",
      "harness": "Mini-SWE-Agent",
      "coverage": 47.5,
      "coverage_ci": [
        39.7,
        55.2
      ],
      "precision": 51.1,
      "precision_ci": [
        42.9,
        59.1
      ],
      "tokens": 498000,
      "blocks": 1.4
    }
  },
  "evidence": [
    {
      "model": "GPT-6 Astra",
      "harness": "Mini-SWE-Agent",
      "label": "Causal method named",
      "stack": "No internal stack trace",
      "incidents": 16,
      "coverage": 94.8,
      "coverage_ci": [
        86.5,
        100.0
      ]
    },
    {
      "model": "GPT-6 Astra",
      "harness": "Mini-SWE-Agent",
      "label": "No causal code named",
      "stack": "No internal stack trace",
      "incidents": 42,
      "coverage": 65.0,
      "coverage_ci": [
        52.8,
        76.7
      ]
    },
    {
      "model": "GPT-6 Astra",
      "harness": "Mini-SWE-Agent",
      "label": "No causal code named",
      "stack": "Cross-component crash site",
      "incidents": 7,
      "coverage": 38.1,
      "coverage_ci": [
        9.5,
        71.4
      ]
    }
  ],
  "harnesses": [
    {
      "name": "Mini-SWE-Agent",
      "coverage": 56.4,
      "coverage_ci": [
        48.5,
        64.2
      ],
      "precision": 43.1,
      "precision_ci": [
        36.6,
        49.5
      ],
      "blocks": 2.3
    },
    {
      "name": "Codex CLI",
      "coverage": 46.5,
      "coverage_ci": [
        38.6,
        54.5
      ],
      "precision": 49.3,
      "precision_ci": [
        41.2,
        57.4
      ],
      "blocks": 1.6
    },
    {
      "name": "OpenCode",
      "coverage": 50.0,
      "coverage_ci": [
        42.4,
        57.9
      ],
      "precision": 47.7,
      "precision_ci": [
        40.2,
        55.4
      ],
      "blocks": 1.8
    }
  ],
  "token_comparison": {
    "harness": "Mini-SWE-Agent",
    "incidents": 120,
    "selection": "Four illustrative models: two recent GPT models and two higher-token models. Full configurations remain available in All metrics. This is not a representative sample or controlled token-budget experiment.",
    "models": [
      {
        "name": "GPT-5.5",
        "family": "openai",
        "tokens": 468000,
        "coverage": 65.1,
        "coverage_ci": [
          57.8,
          72.3
        ]
      },
      {
        "name": "GPT-5.4",
        "family": "openai",
        "tokens": 690000,
        "coverage": 63.1,
        "coverage_ci": [
          56.0,
          70.2
        ]
      },
      {
        "name": "DeepSeek V4 Flash",
        "family": "deepseek",
        "tokens": 2291000,
        "coverage": 56.4,
        "coverage_ci": [
          49.0,
          63.7
        ]
      },
      {
        "name": "Kimi K2.6",
        "family": "kimi",
        "tokens": 2796000,
        "coverage": 58.3,
        "coverage_ci": [
          50.9,
          65.6
        ]
      }
    ]
  },
  "cost_comparison": {
    "heading": "Token price is only part of the cost.",
    "interpretation": "Using official model-provider rates, lower per-token prices do not always imply lower cost per diagnosis: Kimi K2.6 is about twice GPT-5.4 here. DeepSeek V4 Flash remains inexpensive even before cache discounts. This compares model usage at published rates, not the causal effect of increasing reasoning effort.",
    "pricing_checked": "2026-10-26",
    "currency": "USD",
    "unit": "per million tokens",
    "harness": "Mini-SWE-Agent",
    "incidents": 120,
    "usage_source": "Selected session_stats.json records referenced by data/eval_attempt_selection.json and data/eval_summary.json in dsd-bench-neurips-ed-2026",
    "aggregation": "Average selected valid attempts within each incident, then weight all 120 incidents equally. No missing attempts imputed.",
    "formula": "((input_tokens - input_cached_tokens) * input_rate + input_cached_tokens * cached_input_rate + output_tokens * output_rate) / 1000000",
    "reasoning": "Output tokens include reasoning; do not add the reasoning subfield again. Kimi API reference: https://platform.kimi.ai/docs/api/chat",
    "scope": "LLM token charges only; excludes taxes, hosting, paid tools, unselected attempts and provider-specific discounts. Recorded usage is held fixed when applying rate scenarios.",
    "billing_limit": "All costs are estimates at official model-provider rates, not reconstructions of historical provider invoices. DeepSeek cache hits were not recorded, so no cache discount is credited. The source cost fields of zero are not interpreted as zero-dollar costs.",
    "long_context": "All audited GPT-5.4/5.5 input calls are below the 272K long-context surcharge threshold.",
    "uncertainty": "Token and cost CIs newly computed from the same 120 per-incident means: 10,000 bug-bootstrap resamples, PCG64 seed 2026, linear 2.5/97.5 percentiles. They quantify incident-sampling uncertainty in the means, not variation of individual runs. Cost rates are fixed; intervals exclude price/provider uncertainty. Coverage CIs are from the paper.",
    "selection": "GPT-5.4, GPT-5.5, DeepSeek V4 Flash and Kimi K2.6; all use Mini-SWE-Agent on the same 120 incidents.",
    "models": [
      {
        "name": "GPT-5.4",
        "experiment_id": "mini_swe_agent_v2_gpt_5_4_medium_effort_combined",
        "family": "openai",
        "provider_price_scenario": "OpenAI Standard",
        "price_source": "https://developers.openai.com/api/docs/models/gpt-5.4",
        "unit_rates": {
          "input": 2.5,
          "cached_input": 0.25,
          "output": 15
        },
        "cost_usd": 0.4925477555555556,
        "cost_ci": [
          0.4400374946875,
          0.5485402681597223
        ],
        "coverage": 63.1,
        "coverage_ci": [
          56.0,
          70.2
        ],
        "provisional": false,
        "mean_usage": {
          "input_tokens": 683196.9388888889,
          "input_cached_tokens": 584547.9111111112,
          "uncached_input_tokens": 98649.02777777778,
          "output_tokens": 6652.547222222222,
          "total_tokens": 689849.4861111111
        },
        "cost_components_usd": {
          "uncached_input": 0.24662256944444444,
          "cached_input": 0.1461369777777778,
          "output": 0.09978820833333334
        },
        "recorded_routes": {
          "OpenAI model prefix; endpoint not explicitly recorded": 360
        },
        "selected_attempts": 360,
        "max_input_tokens_per_call": 133859,
        "tokens_ci": [
          586231.2030555555,
          804918.5915277777
        ]
      },
      {
        "name": "GPT-5.5",
        "experiment_id": "mini_swe_agent_v2_gpt_5_5_medium_effort_fixed",
        "family": "openai",
        "provider_price_scenario": "OpenAI Standard",
        "price_source": "https://developers.openai.com/api/docs/models/gpt-5.5",
        "unit_rates": {
          "input": 5,
          "cached_input": 0.5,
          "output": 30
        },
        "cost_usd": 0.6512536833333333,
        "cost_ci": [
          0.5824254871527778,
          0.7230096890972222
        ],
        "coverage": 65.1,
        "coverage_ci": [
          57.8,
          72.3
        ],
        "provisional": false,
        "mean_usage": {
          "input_tokens": 464311.63333333336,
          "input_cached_tokens": 399027.2,
          "uncached_input_tokens": 65284.433333333334,
          "output_tokens": 4177.263888888889,
          "total_tokens": 468488.89722222224
        },
        "cost_components_usd": {
          "uncached_input": 0.3264221666666667,
          "cached_input": 0.1995136,
          "output": 0.12531791666666667
        },
        "recorded_routes": {
          "OpenAI model prefix; endpoint not explicitly recorded": 360
        },
        "selected_attempts": 360,
        "max_input_tokens_per_call": 101756,
        "tokens_ci": [
          397998.63444444444,
          543992.8079166667
        ]
      },
      {
        "name": "DeepSeek V4 Flash",
        "experiment_id": "mini_swe_agent_deepseek_v4_flash_azure",
        "family": "deepseek",
        "provider_price_scenario": "Official published original V4 Flash rates; no cache discount",
        "price_source": "https://www.deepseek.com/en/platform/",
        "price_crosscheck": "https://api-docs.deepseek.com/quick_start/agent_integrations/pi_mono/",
        "unit_rates": {
          "input": 0.14,
          "cached_input": 0.0028,
          "output": 0.28
        },
        "cost_usd": 0.3233394235555556,
        "cost_ci": [
          0.2606959635777778,
          0.39149308287638895
        ],
        "coverage": 56.4,
        "coverage_ci": [
          49.0,
          63.7
        ],
        "provisional": false,
        "cache_unavailable": true,
        "mean_usage": {
          "input_tokens": 2272275.15,
          "output_tokens": 18646.080555555556,
          "total_tokens": 2290921.2305555553
        },
        "input_cached_tokens": null,
        "cache_assumption": "No cache discount credited because source responses did not report cache-hit usage. This is a conservative estimate at published original-V4 rates, not a claim that cache hits were zero.",
        "cost_components_usd": {
          "uncached_input": 0.318118521,
          "cached_input": 0,
          "output": 0.005220902555555556
        },
        "recorded_routes": {
          "Azure endpoint": 360
        },
        "selected_attempts": 360,
        "max_input_tokens_per_call": 197404,
        "tokens_ci": [
          1845530.6072916668,
          2775357.182708333
        ]
      },
      {
        "name": "Kimi K2.6",
        "experiment_id": "mini_swe_agent_kimi_k2_6",
        "family": "kimi",
        "provider_price_scenario": "Moonshot direct",
        "price_source": "https://platform.kimi.ai/docs/pricing/chat#k2-series-models",
        "unit_rates": {
          "input": 0.95,
          "cached_input": 0.16,
          "output": 4
        },
        "cost_usd": 0.9791371300277778,
        "cost_ci": [
          0.6797712563923611,
          1.3441894433826387
        ],
        "coverage": 58.3,
        "coverage_ci": [
          50.9,
          65.6
        ],
        "provisional": false,
        "mean_usage": {
          "input_tokens": 2771751.425,
          "input_cached_tokens": 2217994.5166666666,
          "uncached_input_tokens": 553756.9083333333,
          "output_tokens": 24547.23611111111,
          "total_tokens": 2796298.6611111113
        },
        "cost_components_usd": {
          "uncached_input": 0.5260690629166667,
          "cached_input": 0.35487912266666666,
          "output": 0.09818894444444444
        },
        "recorded_routes": {
          "OpenRouter": 60,
          "Azure endpoint": 300
        },
        "selected_attempts": 360,
        "max_input_tokens_per_call": 254557,
        "tokens_ci": [
          2042473.534513889,
          3646545.5086111105
        ]
      }
    ],
    "reasoning_detail_audit": "Kimi reasoning-token details exceed reported completion tokens in 590 calls and five aggregate runs. The calculation uses provider-reported completion tokens once. Replacing completion with max(completion, reasoning detail) per call changes mean estimated cost by only $0.000770; the qualitative comparison is unchanged.",
    "audit_code": "websites/analysis/cost-comparison/audit_costs.py (local source audit; not shipped with the website)",
    "price_version_note": "DeepSeek uses official published rates for the evaluated V4 Flash model. The current API alias serves V4.1 Flash, whose pricing is not substituted. Other rates verified October 26, 2026.",
    "tokens_definition": "Total input plus output tokens per diagnosis, including cached input once and reasoning within output. Counts and intervals average selected attempts within each incident, then weight all 120 incidents equally."
  }
}