{ "version": 1, "kind": "conditional-gross-program-replay-percentages", "trajectories": 2498, "uniqueIssues": 500, "denominator": { "calls": 129846, "callsWithUsage": 129846, "inputTokens": 3201225688, "inputTokensReportedCalls": 129846, "outputTokens": 42377608, "outputTokensReportedCalls": 129846, "totalTokens": 3243603296, "totalTokensReportedCalls": 129846, "cachedInputTokens": 2498865874, "cachedInputTokensReportedCalls": 121228, "cacheCreationInputTokens": 31547900, "cacheCreationInputTokensReportedCalls": 40589, "reasoningOutputTokens": 23075825, "reasoningOutputTokensReportedCalls": 129846, "sourceReportedCostSum": 1344.5910701999996, "callsWithReportedCost": 129846 }, "programs": [ { "program": "bounded_read_continuation_v4", "title": "Continue a bounded source read", "round": "previous", "proposals": 12335, "proposalMatchPercent": 0.3080664775030401, "matchedUsage": { "calls": 38, "callsWithUsage": 38, "inputTokens": 515742, "inputTokensReportedCalls": 38, "outputTokens": 2592, "outputTokensReportedCalls": 38, "totalTokens": 518334, "totalTokensReportedCalls": 38, "cachedInputTokens": 356098, "cachedInputTokensReportedCalls": 36, "cacheCreationInputTokens": 10620, "cacheCreationInputTokensReportedCalls": 9, "reasoningOutputTokens": 576, "reasoningOutputTokensReportedCalls": 38, "sourceReportedCostSum": 0.30480796, "callsWithReportedCost": 38 }, "matchedIssues": 36, "callPercent": 0.029265437518290898, "tokenPercent": 0.015980190938861348, "sourceReportedCostPercent": 0.022669194133102616 }, { "program": "edit_compile_v3", "title": "Check syntax after a source edit", "round": "previous", "proposals": 1574, "proposalMatchPercent": 1.9059720457433291, "matchedUsage": { "calls": 30, "callsWithUsage": 30, "inputTokens": 947946, "inputTokensReportedCalls": 30, "outputTokens": 1403, "outputTokensReportedCalls": 30, "totalTokens": 949349, "totalTokensReportedCalls": 30, "cachedInputTokens": 815104, "cachedInputTokensReportedCalls": 30, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 372, "reasoningOutputTokensReportedCalls": 30, "sourceReportedCostSum": 0.3947587, "callsWithReportedCost": 30 }, "matchedIssues": 26, "callPercent": 0.023104292777598078, "tokenPercent": 0.029268344904283882, "sourceReportedCostPercent": 0.029359015447074335 }, { "program": "edited_file_patch_review_v4", "title": "Review the edited file diff", "round": "previous", "proposals": 1574, "proposalMatchPercent": 0.25412960609911056, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 121735, "inputTokensReportedCalls": 4, "outputTokens": 225, "outputTokensReportedCalls": 4, "totalTokens": 121960, "totalTokensReportedCalls": 4, "cachedInputTokens": 110488, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 267, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 65, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.0251073, "callsWithReportedCost": 4 }, "matchedIssues": 4, "callPercent": 0.0030805723703464104, "tokenPercent": 0.003760015910404353, "sourceReportedCostPercent": 0.0018672814773539616 }, { "program": "empty_search_parent_scope_v5", "title": "Broaden an empty search to the parent directory", "round": "previous", "proposals": 784, "proposalMatchPercent": 1.7857142857142858, "matchedUsage": { "calls": 14, "callsWithUsage": 14, "inputTokens": 395884, "inputTokensReportedCalls": 14, "outputTokens": 805, "outputTokensReportedCalls": 14, "totalTokens": 396689, "totalTokensReportedCalls": 14, "cachedInputTokens": 267426, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 395, "reasoningOutputTokensReportedCalls": 14, "sourceReportedCostSum": 0.07653624, "callsWithReportedCost": 14 }, "matchedIssues": 13, "callPercent": 0.010782003296212437, "tokenPercent": 0.012229886450331193, "sourceReportedCostPercent": 0.005692157392404496 }, { "program": "explicit_finish_after_verification_v5", "title": "Finish under an explicit verification policy", "round": "previous", "proposals": 0, "proposalMatchPercent": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0 }, { "program": "inspect_unique_small_file_v3", "title": "Read a uniquely identified small file", "round": "previous", "proposals": 4191, "proposalMatchPercent": 0.7396802672393223, "matchedUsage": { "calls": 31, "callsWithUsage": 31, "inputTokens": 330311, "inputTokensReportedCalls": 31, "outputTokens": 2952, "outputTokensReportedCalls": 31, "totalTokens": 333263, "totalTokensReportedCalls": 31, "cachedInputTokens": 213492, "cachedInputTokensReportedCalls": 26, "cacheCreationInputTokens": 878, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 1528, "reasoningOutputTokensReportedCalls": 31, "sourceReportedCostSum": 0.09275051999999999, "callsWithReportedCost": 31 }, "matchedIssues": 30, "callPercent": 0.02387443587018468, "tokenPercent": 0.01027446853352809, "sourceReportedCostPercent": 0.006898046703984425 }, { "program": "known_path_read_repair_v5", "title": "Repair a read using a uniquely known path", "round": "previous", "proposals": 0, "proposalMatchPercent": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0 }, { "program": "missing_pytest_module_retry_v5", "title": "Retry a missing pytest executable through its module", "round": "previous", "proposals": 106, "proposalMatchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0 }, { "program": "missing_rg_literal_search_repair_v3", "title": "Repair a missing ripgrep command", "round": "previous", "proposals": 11, "proposalMatchPercent": 81.81818181818181, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 35540, "inputTokensReportedCalls": 9, "outputTokens": 506, "outputTokensReportedCalls": 9, "totalTokens": 36046, "totalTokensReportedCalls": 9, "cachedInputTokens": 19456, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 116, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.0386358, "callsWithReportedCost": 9 }, "matchedIssues": 9, "callPercent": 0.006931287833279423, "tokenPercent": 0.0011112949615155404, "sourceReportedCostPercent": 0.0028734238131042445 }, { "program": "registered_failed_test_rerun_v2", "title": "Rerun a failed assertion regression", "round": "previous", "proposals": 3, "proposalMatchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0 }, { "program": "registered_test_rerun_v2", "title": "Rerun a previously passing test", "round": "previous", "proposals": 92, "proposalMatchPercent": 19.565217391304348, "matchedUsage": { "calls": 18, "callsWithUsage": 18, "inputTokens": 444361, "inputTokensReportedCalls": 18, "outputTokens": 679, "outputTokensReportedCalls": 18, "totalTokens": 445040, "totalTokensReportedCalls": 18, "cachedInputTokens": 262107, "cachedInputTokensReportedCalls": 11, "cacheCreationInputTokens": 138, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 254, "reasoningOutputTokensReportedCalls": 18, "sourceReportedCostSum": 0.11198141, "callsWithReportedCost": 18 }, "matchedIssues": 17, "callPercent": 0.013862575666558847, "tokenPercent": 0.013720543463154749, "sourceReportedCostPercent": 0.008328287498097355 }, { "program": "search_read_v2", "title": "Read context around the only search hit", "round": "previous", "proposals": 3572, "proposalMatchPercent": 0.083986562150056, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 13897, "inputTokensReportedCalls": 3, "outputTokens": 231, "outputTokensReportedCalls": 3, "totalTokens": 14128, "totalTokensReportedCalls": 3, "cachedInputTokens": 957, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 66, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.00359573, "callsWithReportedCost": 3 }, "matchedIssues": 3, "callPercent": 0.002310429277759808, "tokenPercent": 0.00043556497853552555, "sourceReportedCostPercent": 0.00026742182658294445 }, { "program": "syntax_check_status_v3", "title": "Inspect status after a syntax check", "round": "previous", "proposals": 382, "proposalMatchPercent": 10.99476439790576, "matchedUsage": { "calls": 42, "callsWithUsage": 42, "inputTokens": 1101960, "inputTokensReportedCalls": 42, "outputTokens": 11742, "outputTokensReportedCalls": 42, "totalTokens": 1113702, "totalTokensReportedCalls": 42, "cachedInputTokens": 1006720, "cachedInputTokensReportedCalls": 42, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 10445, "reasoningOutputTokensReportedCalls": 42, "sourceReportedCostSum": 0.5072339999999999, "callsWithReportedCost": 42 }, "matchedIssues": 42, "callPercent": 0.03234600988863731, "tokenPercent": 0.03433533321949122, "sourceReportedCostPercent": 0.03772403455903897 }, { "program": "traceback_context_v4", "title": "Read source context from a traceback", "round": "previous", "proposals": 862, "proposalMatchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0 }, { "program": "write_then_run_v2", "title": "Write a script and run it", "round": "previous", "proposals": 2238, "proposalMatchPercent": 19.571045576407506, "matchedUsage": { "calls": 438, "callsWithUsage": 438, "inputTokens": 6984083, "inputTokensReportedCalls": 438, "outputTokens": 33221, "outputTokensReportedCalls": 438, "totalTokens": 7017304, "totalTokensReportedCalls": 438, "cachedInputTokens": 5812335, "cachedInputTokensReportedCalls": 438, "cacheCreationInputTokens": 310874, "cacheCreationInputTokensReportedCalls": 343, "reasoningOutputTokens": 4062, "reasoningOutputTokensReportedCalls": 438, "sourceReportedCostSum": 3.402355650000003, "callsWithReportedCost": 438 }, "matchedIssues": 166, "callPercent": 0.3373226745529319, "tokenPercent": 0.21634285575716716, "sourceReportedCostPercent": 0.25304017893662817 }, { "program": "workflow_repro_v6", "title": "Run a fresh reproduction and classify its failure", "round": "new", "sourceReport": "workflow-repro-v6.json", "proposals": 1352, "proposalMatchPercent": 25.88757396449704, "matchedUsage": { "calls": 350, "callsWithUsage": 350, "inputTokens": 4813857, "inputTokensReportedCalls": 350, "outputTokens": 27587, "outputTokensReportedCalls": 350, "totalTokens": 4841444, "totalTokensReportedCalls": 350, "cachedInputTokens": 4315464, "cachedInputTokensReportedCalls": 350, "cacheCreationInputTokens": 282296, "cacheCreationInputTokensReportedCalls": 320, "reasoningOutputTokens": 3473, "reasoningOutputTokensReportedCalls": 350, "sourceReportedCostSum": 2.7956321900000023, "callsWithReportedCost": 350 }, "matchedIssues": 141, "callPercent": 0.2695500824053109, "tokenPercent": 0.1492612862359109, "sourceReportedCostPercent": 0.20791690886242234, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 889, "matchedCalls": 30, "uniqueMatchedCalls": 30, "matchedIssues": 22, "matchedUsage": { "calls": 30, "callsWithUsage": 30, "inputTokens": 378735, "inputTokensReportedCalls": 30, "outputTokens": 1687, "outputTokensReportedCalls": 30, "totalTokens": 380422, "totalTokensReportedCalls": 30, "cachedInputTokens": 163949, "cachedInputTokensReportedCalls": 30, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 127, "reasoningOutputTokensReportedCalls": 30, "sourceReportedCostSum": 0.08154669, "callsWithReportedCost": 30 }, "callPercent": 0.0686766018817389, "tokenPercent": 0.03193676756763134, "completeTwoStepChains": 0, "secondStageMatches": 0, "secondStageUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnly": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "diagnostics": { "environment-or-import-failure": 15, "completed": 11, "nonzero-without-traceback": 1, "application-traceback": 3, "contextProposals": 2 } }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 1, "matchedCalls": 0, "uniqueMatchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "callPercent": 0.0, "tokenPercent": 0.0, "completeTwoStepChains": 0, "secondStageMatches": 0, "secondStageUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnly": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "diagnostics": {} }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 0, "matchedCalls": 0, "uniqueMatchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "callPercent": 0.0, "tokenPercent": 0.0, "completeTwoStepChains": 0, "secondStageMatches": 0, "secondStageUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnly": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "diagnostics": {} }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 14, "matchedCalls": 4, "uniqueMatchedCalls": 4, "matchedIssues": 3, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 42861, "inputTokensReportedCalls": 4, "outputTokens": 313, "outputTokensReportedCalls": 4, "totalTokens": 43174, "totalTokensReportedCalls": 4, "cachedInputTokens": 40295, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 2534, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 41, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.04397, "callsWithReportedCost": 4 }, "callPercent": 0.024322023592362886, "tokenPercent": 0.012871304405278911, "completeTwoStepChains": 0, "secondStageMatches": 0, "secondStageUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnly": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "diagnostics": { "environment-or-import-failure": 3, "completed": 1 } }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 448, "matchedCalls": 316, "uniqueMatchedCalls": 316, "matchedIssues": 126, "matchedUsage": { "calls": 316, "callsWithUsage": 316, "inputTokens": 4392261, "inputTokensReportedCalls": 316, "outputTokens": 25587, "outputTokensReportedCalls": 316, "totalTokens": 4417848, "totalTokensReportedCalls": 316, "cachedInputTokens": 4111220, "cachedInputTokensReportedCalls": 316, "cacheCreationInputTokens": 279762, "cacheCreationInputTokensReportedCalls": 316, "reasoningOutputTokens": 3305, "reasoningOutputTokensReportedCalls": 316, "sourceReportedCostSum": 2.6701155000000014, "callsWithReportedCost": 316 }, "callPercent": 1.3088679948639357, "tokenPercent": 0.834007514175411, "completeTwoStepChains": 0, "secondStageMatches": 0, "secondStageUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnly": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "diagnostics": { "environment-or-import-failure": 131, "completed": 163, "application-traceback": 19, "contextProposals": 11, "nonzero-without-traceback": 3 } } }, "partitions": { "discovery": { "trajectories": 505, "issues": 101, "denominator": { "calls": 26844, "callsWithUsage": 26844, "inputTokens": 687396884, "inputTokensReportedCalls": 26844, "outputTokens": 9136292, "outputTokensReportedCalls": 26844, "totalTokens": 696533176, "totalTokensReportedCalls": 26844, "cachedInputTokens": 543857432, "cachedInputTokensReportedCalls": 25125, "cacheCreationInputTokens": 6691255, "cacheCreationInputTokensReportedCalls": 8384, "reasoningOutputTokens": 5022443, "reasoningOutputTokensReportedCalls": 26844, "sourceReportedCostSum": 286.1603561800012, "callsWithReportedCost": 26844 }, "proposals": 358, "matchedCalls": 60, "uniqueMatchedCalls": 60, "matchedIssues": 26, "matchedUsage": { "calls": 60, "callsWithUsage": 60, "inputTokens": 792546, "inputTokensReportedCalls": 60, "outputTokens": 4731, "outputTokensReportedCalls": 60, "totalTokens": 797277, "totalTokensReportedCalls": 60, "cachedInputTokens": 686781, "cachedInputTokensReportedCalls": 60, "cacheCreationInputTokens": 43586, "cacheCreationInputTokensReportedCalls": 55, "reasoningOutputTokens": 610, "reasoningOutputTokensReportedCalls": 60, "sourceReportedCostSum": 0.45331517, "callsWithReportedCost": 60 }, "callPercent": 0.22351363433169424, "tokenPercent": 0.11446360740181025, "completeTwoStepChains": 0, "secondStageMatches": 0, "secondStageUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnly": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "diagnostics": { "application-traceback": 7, "contextProposals": 4, "environment-or-import-failure": 30, "completed": 23 } }, "validation": { "trajectories": 1993, "issues": 399, "denominator": { "calls": 103002, "callsWithUsage": 103002, "inputTokens": 2513828804, "inputTokensReportedCalls": 103002, "outputTokens": 33241316, "outputTokensReportedCalls": 103002, "totalTokens": 2547070120, "totalTokensReportedCalls": 103002, "cachedInputTokens": 1955008442, "cachedInputTokensReportedCalls": 96103, "cacheCreationInputTokens": 24856645, "cacheCreationInputTokensReportedCalls": 32205, "reasoningOutputTokens": 18053382, "reasoningOutputTokensReportedCalls": 103002, "sourceReportedCostSum": 1058.4307140200008, "callsWithReportedCost": 103002 }, "proposals": 994, "matchedCalls": 290, "uniqueMatchedCalls": 290, "matchedIssues": 115, "matchedUsage": { "calls": 290, "callsWithUsage": 290, "inputTokens": 4021311, "inputTokensReportedCalls": 290, "outputTokens": 22856, "outputTokensReportedCalls": 290, "totalTokens": 4044167, "totalTokensReportedCalls": 290, "cachedInputTokens": 3628683, "cachedInputTokensReportedCalls": 290, "cacheCreationInputTokens": 238710, "cacheCreationInputTokensReportedCalls": 265, "reasoningOutputTokens": 2863, "reasoningOutputTokensReportedCalls": 290, "sourceReportedCostSum": 2.342317020000001, "callsWithReportedCost": 290 }, "callPercent": 0.2815479311081338, "tokenPercent": 0.15877721497514172, "completeTwoStepChains": 0, "secondStageMatches": 0, "secondStageUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnly": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "diagnostics": { "environment-or-import-failure": 119, "completed": 152, "nonzero-without-traceback": 4, "application-traceback": 15, "contextProposals": 9 } } }, "workflowEvidence": { "startedExactSequences": 350, "sequencesWithMultipleExactStages": 0, "additionalExactStages": 0, "multiStageOriginalUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "addedToConservativeUnion": false, "netSavingsDemonstrated": false } }, { "program": "guarded_edit_compile_focused_test_v6", "title": "Compile an edited file, then rerun a registered focused test", "round": "new", "sourceReport": "workflow-verify-v6.json", "proposals": 84, "proposalMatchPercent": 1.1904761904761905, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 28096, "inputTokensReportedCalls": 1, "outputTokens": 33, "outputTokensReportedCalls": 1, "totalTokens": 28129, "totalTokensReportedCalls": 1, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.049629999999999994, "callsWithReportedCost": 1 }, "matchedIssues": 1, "callPercent": 0.0007701430925866026, "tokenPercent": 0.0008672145584106596, "sourceReportedCostPercent": 0.0036910850518007557, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 3, "workflowStarts": 3, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "proposalMatchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "firstStepOverlap": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "newSecondStage": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "completeSequentialChains": { "chains": 0, "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "testExitCodes": {} }, "unmatchedProposedSteps": 3, "maxHypotheticalToolCallsIfEveryStartCompilesSuccessfully": 6, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSuccessfulSourceEdits": 530, "noRecentSuccessfulFocusedRegistrationScenario": 527, "nextNotExactCompile": 3 } }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 57, "workflowStarts": 57, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "proposalMatchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "firstStepOverlap": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "newSecondStage": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "completeSequentialChains": { "chains": 0, "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "testExitCodes": {} }, "unmatchedProposedSteps": 57, "maxHypotheticalToolCallsIfEveryStartCompilesSuccessfully": 114, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSuccessfulSourceEdits": 232, "noRecentSuccessfulFocusedRegistrationScenario": 175, "nextNotExactCompile": 57 } }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 2, "workflowStarts": 1, "matchedCalls": 1, "matchedIssues": 1, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 28096, "inputTokensReportedCalls": 1, "outputTokens": 33, "outputTokensReportedCalls": 1, "totalTokens": 28129, "totalTokensReportedCalls": 1, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.049629999999999994, "callsWithReportedCost": 1 }, "proposalMatchPercent": 50.0, "callPercent": 0.005707762557077625, "tokenPercent": 0.008542293702703624, "firstStepOverlap": { "calls": 1, "usage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 28096, "inputTokensReportedCalls": 1, "outputTokens": 33, "outputTokensReportedCalls": 1, "totalTokens": 28129, "totalTokensReportedCalls": 1, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.049629999999999994, "callsWithReportedCost": 1 } }, "newSecondStage": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "completeSequentialChains": { "chains": 0, "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "testExitCodes": {} }, "unmatchedProposedSteps": 1, "maxHypotheticalToolCallsIfEveryStartCompilesSuccessfully": 2, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSuccessfulSourceEdits": 557, "noRecentSuccessfulFocusedRegistrationScenario": 556, "nextAfterCompileNotRegisteredTest": 1 } }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 7, "workflowStarts": 7, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "proposalMatchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "firstStepOverlap": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "newSecondStage": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "completeSequentialChains": { "chains": 0, "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "testExitCodes": {} }, "unmatchedProposedSteps": 7, "maxHypotheticalToolCallsIfEveryStartCompilesSuccessfully": 14, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSuccessfulSourceEdits": 188, "noRecentSuccessfulFocusedRegistrationScenario": 181, "nextNotExactCompile": 7 } }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 15, "workflowStarts": 15, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "proposalMatchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "firstStepOverlap": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "newSecondStage": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "completeSequentialChains": { "chains": 0, "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "testExitCodes": {} }, "unmatchedProposedSteps": 15, "maxHypotheticalToolCallsIfEveryStartCompilesSuccessfully": 30, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSuccessfulSourceEdits": 67, "noRecentSuccessfulFocusedRegistrationScenario": 52, "nextNotExactCompile": 15 } } }, "partitions": { "discovery": { "trajectories": 505, "issues": 101, "denominator": { "calls": 26844, "callsWithUsage": 26844, "inputTokens": 687396884, "inputTokensReportedCalls": 26844, "outputTokens": 9136292, "outputTokensReportedCalls": 26844, "totalTokens": 696533176, "totalTokensReportedCalls": 26844, "cachedInputTokens": 543857432, "cachedInputTokensReportedCalls": 25125, "cacheCreationInputTokens": 6691255, "cacheCreationInputTokensReportedCalls": 8384, "reasoningOutputTokens": 5022443, "reasoningOutputTokensReportedCalls": 26844, "sourceReportedCostSum": 286.1603561800012, "callsWithReportedCost": 26844 }, "proposals": 17, "workflowStarts": 17, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "proposalMatchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "firstStepOverlap": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "newSecondStage": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "completeSequentialChains": { "chains": 0, "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "testExitCodes": {} }, "unmatchedProposedSteps": 17, "maxHypotheticalToolCallsIfEveryStartCompilesSuccessfully": 34, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSuccessfulSourceEdits": 295, "noRecentSuccessfulFocusedRegistrationScenario": 278, "nextNotExactCompile": 17 } }, "validation": { "trajectories": 1993, "issues": 399, "denominator": { "calls": 103002, "callsWithUsage": 103002, "inputTokens": 2513828804, "inputTokensReportedCalls": 103002, "outputTokens": 33241316, "outputTokensReportedCalls": 103002, "totalTokens": 2547070120, "totalTokensReportedCalls": 103002, "cachedInputTokens": 1955008442, "cachedInputTokensReportedCalls": 96103, "cacheCreationInputTokens": 24856645, "cacheCreationInputTokensReportedCalls": 32205, "reasoningOutputTokens": 18053382, "reasoningOutputTokensReportedCalls": 103002, "sourceReportedCostSum": 1058.4307140200008, "callsWithReportedCost": 103002 }, "proposals": 67, "workflowStarts": 66, "matchedCalls": 1, "matchedIssues": 1, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 28096, "inputTokensReportedCalls": 1, "outputTokens": 33, "outputTokensReportedCalls": 1, "totalTokens": 28129, "totalTokensReportedCalls": 1, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.049629999999999994, "callsWithReportedCost": 1 }, "proposalMatchPercent": 1.492537313432836, "callPercent": 0.0009708549348556338, "tokenPercent": 0.001104366926498278, "firstStepOverlap": { "calls": 1, "usage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 28096, "inputTokensReportedCalls": 1, "outputTokens": 33, "outputTokensReportedCalls": 1, "totalTokens": 28129, "totalTokensReportedCalls": 1, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.049629999999999994, "callsWithReportedCost": 1 } }, "newSecondStage": { "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "completeSequentialChains": { "chains": 0, "calls": 0, "usage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "testExitCodes": {} }, "unmatchedProposedSteps": 66, "maxHypotheticalToolCallsIfEveryStartCompilesSuccessfully": 132, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSuccessfulSourceEdits": 1279, "noRecentSuccessfulFocusedRegistrationScenario": 1213, "nextNotExactCompile": 65, "nextAfterCompileNotRegisteredTest": 1 } } }, "workflowEvidence": { "startedExactSequences": 1, "sequencesWithMultipleExactStages": 0, "additionalExactStages": 0, "multiStageOriginalUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "addedToConservativeUnion": false, "netSavingsDemonstrated": false } }, { "program": "workflow_investigate_v6", "title": "Investigate a unique Python definition and expand proven cutoffs", "round": "new", "sourceReport": "workflow-investigate-v6.json", "proposals": 2384, "proposalMatchPercent": 5.578859060402684, "matchedUsage": { "calls": 133, "callsWithUsage": 133, "inputTokens": 2133059, "inputTokensReportedCalls": 133, "outputTokens": 9864, "outputTokensReportedCalls": 133, "totalTokens": 2142923, "totalTokensReportedCalls": 133, "cachedInputTokens": 1269060, "cachedInputTokensReportedCalls": 114, "cacheCreationInputTokens": 6117, "cacheCreationInputTokensReportedCalls": 30, "reasoningOutputTokens": 1986, "reasoningOutputTokensReportedCalls": 133, "sourceReportedCostSum": 0.6097669399999998, "callsWithReportedCost": 133 }, "matchedIssues": 114, "callPercent": 0.10242903131401815, "tokenPercent": 0.06606612475214355, "sourceReportedCostPercent": 0.04534961993383614, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 933, "matchedCalls": 64, "matchedIssues": 59, "matchedUsage": { "calls": 64, "callsWithUsage": 64, "inputTokens": 1060156, "inputTokensReportedCalls": 64, "outputTokens": 4593, "outputTokensReportedCalls": 64, "totalTokens": 1064749, "totalTokensReportedCalls": 64, "cachedInputTokens": 564882, "cachedInputTokensReportedCalls": 64, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 600, "reasoningOutputTokensReportedCalls": 64, "sourceReportedCostSum": 0.21568423999999994, "callsWithReportedCost": 64 }, "precision": 0.06859592711682744, "callPercent": 0.1465100840143763, "tokenPercent": 0.08938663203197476, "completeSequences": 0, "completeSequenceUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "diagnostics": { "stage1Proposals": 919, "stage1Matches": 64, "coverageOnlyCalls": 154, "coverageOnlyTokens": 3648317, "stage2Proposals": 14 }, "overlap": { "searchReadV2": 0, "readContinuationV4": 0 } }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 893, "matchedCalls": 39, "matchedIssues": 37, "matchedUsage": { "calls": 39, "callsWithUsage": 39, "inputTokens": 725884, "inputTokensReportedCalls": 39, "outputTokens": 2425, "outputTokensReportedCalls": 39, "totalTokens": 728309, "totalTokensReportedCalls": 39, "cachedInputTokens": 363435, "cachedInputTokensReportedCalls": 20, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 1025, "reasoningOutputTokensReportedCalls": 39, "sourceReportedCostSum": 0.20667124999999992, "callsWithReportedCost": 39 }, "precision": 0.043673012318029114, "callPercent": 0.13901760889712697, "tokenPercent": 0.08488466781236903, "completeSequences": 0, "completeSequenceUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "diagnostics": { "stage1Proposals": 883, "coverageOnlyCalls": 291, "coverageOnlyTokens": 6748139, "stage1Matches": 39, "stage2Proposals": 10 }, "overlap": { "searchReadV2": 0, "readContinuationV4": 0 } }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 279, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "precision": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "completeSequences": 0, "completeSequenceUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "diagnostics": { "stage1Proposals": 279, "coverageOnlyCalls": 9, "coverageOnlyTokens": 181216 }, "overlap": { "searchReadV2": 0, "readContinuationV4": 0 } }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 42, "matchedCalls": 4, "matchedIssues": 4, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 67100, "inputTokensReportedCalls": 4, "outputTokens": 371, "outputTokensReportedCalls": 4, "totalTokens": 67471, "totalTokensReportedCalls": 4, "cachedInputTokens": 66244, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 824, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 39, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.04770699999999999, "callsWithReportedCost": 4 }, "precision": 0.09523809523809523, "callPercent": 0.024322023592362886, "tokenPercent": 0.020114878851359, "completeSequences": 0, "completeSequenceUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "diagnostics": { "stage1Proposals": 40, "coverageOnlyCalls": 9, "coverageOnlyTokens": 186572, "stage1Matches": 4, "stage2Proposals": 2 }, "overlap": { "searchReadV2": 0, "readContinuationV4": 0 } }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 237, "matchedCalls": 26, "matchedIssues": 25, "matchedUsage": { "calls": 26, "callsWithUsage": 26, "inputTokens": 279919, "inputTokensReportedCalls": 26, "outputTokens": 2475, "outputTokensReportedCalls": 26, "totalTokens": 282394, "totalTokensReportedCalls": 26, "cachedInputTokens": 274499, "cachedInputTokensReportedCalls": 26, "cacheCreationInputTokens": 5293, "cacheCreationInputTokensReportedCalls": 26, "reasoningOutputTokens": 322, "reasoningOutputTokensReportedCalls": 26, "sourceReportedCostSum": 0.13970445, "callsWithReportedCost": 26 }, "precision": 0.10970464135021098, "callPercent": 0.10769167046348838, "tokenPercent": 0.05331073363276668, "completeSequences": 0, "completeSequenceUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "diagnostics": { "stage1Proposals": 232, "coverageOnlyCalls": 53, "stage1Matches": 26, "stage2Proposals": 5, "coverageOnlyTokens": 1042260 }, "overlap": { "searchReadV2": 0, "readContinuationV4": 0 } } }, "partitions": { "discovery": { "trajectories": 505, "issues": 101, "denominator": { "calls": 26844, "callsWithUsage": 26844, "inputTokens": 687396884, "inputTokensReportedCalls": 26844, "outputTokens": 9136292, "outputTokensReportedCalls": 26844, "totalTokens": 696533176, "totalTokensReportedCalls": 26844, "cachedInputTokens": 543857432, "cachedInputTokensReportedCalls": 25125, "cacheCreationInputTokens": 6691255, "cacheCreationInputTokensReportedCalls": 8384, "reasoningOutputTokens": 5022443, "reasoningOutputTokensReportedCalls": 26844, "sourceReportedCostSum": 286.1603561800012, "callsWithReportedCost": 26844 }, "proposals": 492, "matchedCalls": 25, "matchedIssues": 21, "matchedUsage": { "calls": 25, "callsWithUsage": 25, "inputTokens": 412708, "inputTokensReportedCalls": 25, "outputTokens": 1777, "outputTokensReportedCalls": 25, "totalTokens": 414485, "totalTokensReportedCalls": 25, "cachedInputTokens": 252767, "cachedInputTokensReportedCalls": 20, "cacheCreationInputTokens": 1258, "cacheCreationInputTokensReportedCalls": 7, "reasoningOutputTokens": 292, "reasoningOutputTokensReportedCalls": 25, "sourceReportedCostSum": 0.12778134, "callsWithReportedCost": 25 }, "precision": 0.0508130081300813, "callPercent": 0.09313068097153926, "tokenPercent": 0.05950685685644929, "completeSequences": 0, "completeSequenceUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "diagnostics": { "stage1Proposals": 488, "coverageOnlyCalls": 97, "coverageOnlyTokens": 2026515, "stage1Matches": 25, "stage2Proposals": 4 }, "overlap": { "searchReadV2": 0, "readContinuationV4": 0 } }, "validation": { "trajectories": 1993, "issues": 399, "denominator": { "calls": 103002, "callsWithUsage": 103002, "inputTokens": 2513828804, "inputTokensReportedCalls": 103002, "outputTokens": 33241316, "outputTokensReportedCalls": 103002, "totalTokens": 2547070120, "totalTokensReportedCalls": 103002, "cachedInputTokens": 1955008442, "cachedInputTokensReportedCalls": 96103, "cacheCreationInputTokens": 24856645, "cacheCreationInputTokensReportedCalls": 32205, "reasoningOutputTokens": 18053382, "reasoningOutputTokensReportedCalls": 103002, "sourceReportedCostSum": 1058.4307140200008, "callsWithReportedCost": 103002 }, "proposals": 1892, "matchedCalls": 108, "matchedIssues": 93, "matchedUsage": { "calls": 108, "callsWithUsage": 108, "inputTokens": 1720351, "inputTokensReportedCalls": 108, "outputTokens": 8087, "outputTokensReportedCalls": 108, "totalTokens": 1728438, "totalTokensReportedCalls": 108, "cachedInputTokens": 1016293, "cachedInputTokensReportedCalls": 94, "cacheCreationInputTokens": 4859, "cacheCreationInputTokensReportedCalls": 23, "reasoningOutputTokens": 1694, "reasoningOutputTokensReportedCalls": 108, "sourceReportedCostSum": 0.4819855999999999, "callsWithReportedCost": 108 }, "precision": 0.05708245243128964, "callPercent": 0.10485233296440846, "tokenPercent": 0.06785985145944863, "completeSequences": 0, "completeSequenceUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "diagnostics": { "stage1Proposals": 1865, "stage1Matches": 108, "coverageOnlyCalls": 419, "coverageOnlyTokens": 9779989, "stage2Proposals": 27 }, "overlap": { "searchReadV2": 0, "readContinuationV4": 0 } } }, "workflowEvidence": { "startedExactSequences": 133, "sequencesWithMultipleExactStages": 0, "additionalExactStages": 0, "multiStageOriginalUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "addedToConservativeUnion": false, "netSavingsDemonstrated": false } } ], "previousUnion": { "matchedUsage": { "calls": 627, "callsWithUsage": 627, "inputTokens": 10891459, "inputTokensReportedCalls": 627, "outputTokens": 54356, "outputTokensReportedCalls": 627, "totalTokens": 10945815, "totalTokensReportedCalls": 627, "cachedInputTokens": 8864183, "cachedInputTokensReportedCalls": 608, "cacheCreationInputTokens": 322777, "cacheCreationInputTokensReportedCalls": 358, "reasoningOutputTokens": 17879, "reasoningOutputTokensReportedCalls": 627, "sourceReportedCostSum": 4.957763310000003, "callsWithReportedCost": 627 }, "matchedIssues": 264, "callPercent": 0.4828797190517998, "tokenPercent": 0.33745849911727305, "sourceReportedCostPercent": 0.3687190417873716 }, "newRoundUnion": { "matchedUsage": { "calls": 484, "callsWithUsage": 484, "inputTokens": 6975012, "inputTokensReportedCalls": 484, "outputTokens": 37484, "outputTokensReportedCalls": 484, "totalTokens": 7012496, "totalTokensReportedCalls": 484, "cachedInputTokens": 5584524, "cachedInputTokensReportedCalls": 465, "cacheCreationInputTokens": 288413, "cacheCreationInputTokensReportedCalls": 350, "reasoningOutputTokens": 5459, "reasoningOutputTokensReportedCalls": 484, "sourceReportedCostSum": 3.4550291300000033, "callsWithReportedCost": 484 }, "matchedIssues": 226, "callPercent": 0.3727492568119157, "tokenPercent": 0.2161946255464651, "sourceReportedCostPercent": 0.2569576138480593, "duplicateTargetsRemoved": 0, "dependentChainsExcluded": 0 }, "combinedUnion": { "matchedUsage": { "calls": 760, "callsWithUsage": 760, "inputTokens": 13024518, "inputTokensReportedCalls": 760, "outputTokens": 64220, "outputTokensReportedCalls": 760, "totalTokens": 13088738, "totalTokensReportedCalls": 760, "cachedInputTokens": 10133243, "cachedInputTokensReportedCalls": 722, "cacheCreationInputTokens": 328894, "cacheCreationInputTokensReportedCalls": 388, "reasoningOutputTokens": 19865, "reasoningOutputTokensReportedCalls": 760, "sourceReportedCostSum": 5.56753025, "callsWithReportedCost": 760 }, "matchedIssues": 315, "callPercent": 0.585308750365818, "tokenPercent": 0.4035246238694166, "sourceReportedCostPercent": 0.4140686617212075, "duplicateTargetsRemoved": 351, "dependentChainsExcluded": 0 }, "incrementalExactCalls": 133, "sourceManifestSha256": "73cccd8079f5c3150015207a7d102f1a4409c7996f0ea9ae1305f1b45239247b", "verifiedNewEvents": 484, "limitations": [ "Percentages divide matched original calls/tokens by the full corpus denominator, not by proposals.", "Proposal match percentage is descriptive precision, not deployment safety or task success.", "Programs see prior/current observations; future calls are replay labels only.", "Original recorded token counts are known; avoiding those calls is conditional on dispensable reasoning and successful program execution.", "Gross estimates exclude routing/program/false-positive costs and future context/cache effects.", "Per-program savings may overlap; combined totals deduplicate targets and exclude dependent adjacent substitutions.", "New candidates use candidate-specific discovery/validation issue partitions; previously inspected discovery issues are excluded from validation. This corpus is not untouched external validation.", "Coverage-only or EOF-equivalent scenarios are not added to exact totals.", "No recorded commands are executed and no paid model calls occur." ] }