{ "version": 1, "kind": "conditional-gross-program-replay-percentages", "trajectories": 2498, "uniqueIssues": 500, "denominator": { "calls": 129846, "callsWithUsage": 129846, "inputTokens": 3201225688, "inputTokensReportedCalls": 129846, "outputTokens": 42377608, "outputTokensReportedCalls": 129846, "totalTokens": 3243603296, "totalTokensReportedCalls": 129846, "cachedInputTokens": 2498865874, "cachedInputTokensReportedCalls": 121228, "cacheCreationInputTokens": 31547900, "cacheCreationInputTokensReportedCalls": 40589, "reasoningOutputTokens": 23075825, "reasoningOutputTokensReportedCalls": 129846, "sourceReportedCostSum": 1344.5910701999999, "callsWithReportedCost": 129846 }, "programs": [ { "program": "registered_failed_test_rerun_v2", "title": "Rerun a failed assertion regression", "round": "previous", "proposals": 3, "proposalMatchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0 }, { "program": "registered_test_rerun_v2", "title": "Rerun a previously passing test", "round": "previous", "proposals": 92, "proposalMatchPercent": 19.565217391304348, "matchedUsage": { "calls": 18, "callsWithUsage": 18, "inputTokens": 444361, "inputTokensReportedCalls": 18, "outputTokens": 679, "outputTokensReportedCalls": 18, "totalTokens": 445040, "totalTokensReportedCalls": 18, "cachedInputTokens": 262107, "cachedInputTokensReportedCalls": 11, "cacheCreationInputTokens": 138, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 254, "reasoningOutputTokensReportedCalls": 18, "sourceReportedCostSum": 0.11198141, "callsWithReportedCost": 18 }, "matchedIssues": 17, "callPercent": 0.013862575666558847, "tokenPercent": 0.013720543463154749, "sourceReportedCostPercent": 0.008328287498097353 }, { "program": "search_read_v2", "title": "Read context around the only search hit", "round": "previous", "proposals": 3572, "proposalMatchPercent": 0.083986562150056, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 13897, "inputTokensReportedCalls": 3, "outputTokens": 231, "outputTokensReportedCalls": 3, "totalTokens": 14128, "totalTokensReportedCalls": 3, "cachedInputTokens": 957, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 66, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.00359573, "callsWithReportedCost": 3 }, "matchedIssues": 3, "callPercent": 0.002310429277759808, "tokenPercent": 0.00043556497853552555, "sourceReportedCostPercent": 0.00026742182658294445 }, { "program": "write_then_run_v2", "title": "Write a script and run it", "round": "previous", "proposals": 2238, "proposalMatchPercent": 19.571045576407506, "matchedUsage": { "calls": 438, "callsWithUsage": 438, "inputTokens": 6984083, "inputTokensReportedCalls": 438, "outputTokens": 33221, "outputTokensReportedCalls": 438, "totalTokens": 7017304, "totalTokensReportedCalls": 438, "cachedInputTokens": 5812335, "cachedInputTokensReportedCalls": 438, "cacheCreationInputTokens": 310874, "cacheCreationInputTokensReportedCalls": 343, "reasoningOutputTokens": 4062, "reasoningOutputTokensReportedCalls": 438, "sourceReportedCostSum": 3.402355650000003, "callsWithReportedCost": 438 }, "matchedIssues": 166, "callPercent": 0.3373226745529319, "tokenPercent": 0.21634285575716716, "sourceReportedCostPercent": 0.2530401789366281 }, { "program": "inspect_unique_small_file_v3", "title": "Inspect the only matching small file", "round": "new", "sourceReport": "candidate-inspect-file-v3.json", "proposals": 4191, "proposalMatchPercent": 0.7396802672393223, "matchedUsage": { "calls": 31, "callsWithUsage": 31, "inputTokens": 330311, "inputTokensReportedCalls": 31, "outputTokens": 2952, "outputTokensReportedCalls": 31, "totalTokens": 333263, "totalTokensReportedCalls": 31, "cachedInputTokens": 213492, "cachedInputTokensReportedCalls": 26, "cacheCreationInputTokens": 878, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 1528, "reasoningOutputTokensReportedCalls": 31, "sourceReportedCostSum": 0.09275051999999999, "callsWithReportedCost": 31 }, "matchedIssues": 30, "callPercent": 0.02387443587018468, "tokenPercent": 0.01027446853352809, "sourceReportedCostPercent": 0.006898046703984424, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 1820, "matchedCalls": 15, "matchedIssues": 15, "matchedUsage": { "calls": 15, "callsWithUsage": 15, "inputTokens": 111677, "inputTokensReportedCalls": 15, "outputTokens": 1081, "outputTokensReportedCalls": 15, "totalTokens": 112758, "totalTokensReportedCalls": 15, "cachedInputTokens": 48243, "cachedInputTokensReportedCalls": 15, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 265, "reasoningOutputTokensReportedCalls": 15, "sourceReportedCostSum": 0.02414007, "callsWithReportedCost": 15 }, "matchPrecisionPercent": 0.8241758241758241, "callPercent": 0.03433830094086945, "tokenPercent": 0.009466135074709073, "matchesWithReasoningOrProse": 15, "equivalentEOF": { "matchedCalls": 9, "matchedIssues": 9, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 139701, "inputTokensReportedCalls": 9, "outputTokens": 728, "outputTokensReportedCalls": 9, "totalTokens": 140429, "totalTokensReportedCalls": 9, "cachedInputTokens": 76473, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 206, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.028186870000000003, "callsWithReportedCost": 9 }, "excludedFromExactTotal": true } }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 1192, "matchedCalls": 11, "matchedIssues": 11, "matchedUsage": { "calls": 11, "callsWithUsage": 11, "inputTokens": 173788, "inputTokensReportedCalls": 11, "outputTokens": 650, "outputTokensReportedCalls": 11, "totalTokens": 174438, "totalTokensReportedCalls": 11, "cachedInputTokens": 122435, "cachedInputTokensReportedCalls": 6, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 366, "reasoningOutputTokensReportedCalls": 11, "sourceReportedCostSum": 0.03374825, "callsWithReportedCost": 11 }, "matchPrecisionPercent": 0.9228187919463087, "callPercent": 0.03921009481713838, "tokenPercent": 0.020330809702823978, "matchesWithReasoningOrProse": 11, "equivalentEOF": { "matchedCalls": 4, "matchedIssues": 4, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 103848, "inputTokensReportedCalls": 4, "outputTokens": 1134, "outputTokensReportedCalls": 4, "totalTokens": 104982, "totalTokensReportedCalls": 4, "cachedInputTokens": 61380, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 1016, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.027705, "callsWithReportedCost": 4 }, "excludedFromExactTotal": true } }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 660, "matchedCalls": 1, "matchedIssues": 1, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 7274, "inputTokensReportedCalls": 1, "outputTokens": 871, "outputTokensReportedCalls": 1, "totalTokens": 8145, "totalTokensReportedCalls": 1, "cachedInputTokens": 6144, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 838, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.0152467, "callsWithReportedCost": 1 }, "matchPrecisionPercent": 0.15151515151515152, "callPercent": 0.005707762557077625, "tokenPercent": 0.0024734964701383275, "matchesWithReasoningOrProse": 1, "equivalentEOF": { "matchedCalls": 22, "matchedIssues": 21, "matchedUsage": { "calls": 22, "callsWithUsage": 22, "inputTokens": 260605, "inputTokensReportedCalls": 22, "outputTokens": 833, "outputTokensReportedCalls": 22, "totalTokens": 261438, "totalTokensReportedCalls": 22, "cachedInputTokens": 213120, "cachedInputTokensReportedCalls": 22, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 22, "sourceReportedCostSum": 0.13205675, "callsWithReportedCost": 22 }, "excludedFromExactTotal": true } }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 82, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPrecisionPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "equivalentEOF": { "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactTotal": true } }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 437, "matchedCalls": 4, "matchedIssues": 4, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 37572, "inputTokensReportedCalls": 4, "outputTokens": 350, "outputTokensReportedCalls": 4, "totalTokens": 37922, "totalTokensReportedCalls": 4, "cachedInputTokens": 36670, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 878, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 59, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.0196155, "callsWithReportedCost": 4 }, "matchPrecisionPercent": 0.9153318077803204, "callPercent": 0.016567949302075134, "tokenPercent": 0.0071589681112976125, "matchesWithReasoningOrProse": 4, "equivalentEOF": { "matchedCalls": 1, "matchedIssues": 1, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 12080, "inputTokensReportedCalls": 1, "outputTokens": 97, "outputTokensReportedCalls": 1, "totalTokens": 12177, "totalTokensReportedCalls": 1, "cachedInputTokens": 11927, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 153, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 19, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.00560685, "callsWithReportedCost": 1 }, "excludedFromExactTotal": true } } }, "partitions": { "discovery": { "trajectories": 885, "issues": 177, "denominator": { "calls": 45072, "callsWithUsage": 45072, "inputTokens": 1091899494, "inputTokensReportedCalls": 45072, "outputTokens": 14891202, "outputTokensReportedCalls": 45072, "totalTokens": 1106790696, "totalTokensReportedCalls": 45072, "cachedInputTokens": 844764656, "cachedInputTokensReportedCalls": 42040, "cacheCreationInputTokens": 10744848, "cacheCreationInputTokensReportedCalls": 13737, "reasoningOutputTokens": 8047248, "reasoningOutputTokensReportedCalls": 45072, "sourceReportedCostSum": 454.87979764000346, "callsWithReportedCost": 45072 }, "proposals": 1437, "matchedCalls": 8, "matchedIssues": 8, "matchedUsage": { "calls": 8, "callsWithUsage": 8, "inputTokens": 43797, "inputTokensReportedCalls": 8, "outputTokens": 1501, "outputTokensReportedCalls": 8, "totalTokens": 45298, "totalTokensReportedCalls": 8, "cachedInputTokens": 22354, "cachedInputTokensReportedCalls": 6, "cacheCreationInputTokens": 151, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 1137, "reasoningOutputTokensReportedCalls": 8, "sourceReportedCostSum": 0.02898134, "callsWithReportedCost": 8 }, "matchPrecisionPercent": 0.5567153792623522, "callPercent": 0.01774937877174299, "tokenPercent": 0.0040927340791451685, "matchesWithReasoningOrProse": 8, "equivalentEOF": { "matchedCalls": 9, "matchedIssues": 9, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 165510, "inputTokensReportedCalls": 9, "outputTokens": 481, "outputTokensReportedCalls": 9, "totalTokens": 165991, "totalTokensReportedCalls": 9, "cachedInputTokens": 109501, "cachedInputTokensReportedCalls": 8, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 75, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.0875929, "callsWithReportedCost": 9 }, "excludedFromExactTotal": true } }, "validation": { "trajectories": 1613, "issues": 323, "denominator": { "calls": 84774, "callsWithUsage": 84774, "inputTokens": 2109326194, "inputTokensReportedCalls": 84774, "outputTokens": 27486406, "outputTokensReportedCalls": 84774, "totalTokens": 2136812600, "totalTokensReportedCalls": 84774, "cachedInputTokens": 1654101218, "cachedInputTokensReportedCalls": 79188, "cacheCreationInputTokens": 20803052, "cacheCreationInputTokensReportedCalls": 26852, "reasoningOutputTokens": 15028577, "reasoningOutputTokensReportedCalls": 84774, "sourceReportedCostSum": 889.7112725599998, "callsWithReportedCost": 84774 }, "proposals": 2754, "matchedCalls": 23, "matchedIssues": 22, "matchedUsage": { "calls": 23, "callsWithUsage": 23, "inputTokens": 286514, "inputTokensReportedCalls": 23, "outputTokens": 1451, "outputTokensReportedCalls": 23, "totalTokens": 287965, "totalTokensReportedCalls": 23, "cachedInputTokens": 191138, "cachedInputTokensReportedCalls": 20, "cacheCreationInputTokens": 727, "cacheCreationInputTokensReportedCalls": 3, "reasoningOutputTokens": 391, "reasoningOutputTokensReportedCalls": 23, "sourceReportedCostSum": 0.06376918, "callsWithReportedCost": 23 }, "matchPrecisionPercent": 0.8351488743645606, "callPercent": 0.027130959964139948, "tokenPercent": 0.013476380661551697, "matchesWithReasoningOrProse": 23, "equivalentEOF": { "matchedCalls": 27, "matchedIssues": 25, "matchedUsage": { "calls": 27, "callsWithUsage": 27, "inputTokens": 350724, "inputTokensReportedCalls": 27, "outputTokens": 2311, "outputTokensReportedCalls": 27, "totalTokens": 353035, "totalTokensReportedCalls": 27, "cachedInputTokens": 253399, "cachedInputTokensReportedCalls": 27, "cacheCreationInputTokens": 153, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 1166, "reasoningOutputTokensReportedCalls": 27, "sourceReportedCostSum": 0.10596257000000002, "callsWithReportedCost": 27 }, "excludedFromExactTotal": true } } } }, { "program": "syntax_check_status_v3", "title": "Inspect repository status after a syntax check", "round": "new", "sourceReport": "candidate-diff-check-v3.json", "proposals": 382, "proposalMatchPercent": 10.99476439790576, "matchedUsage": { "calls": 42, "callsWithUsage": 42, "inputTokens": 1101960, "inputTokensReportedCalls": 42, "outputTokens": 11742, "outputTokensReportedCalls": 42, "totalTokens": 1113702, "totalTokensReportedCalls": 42, "cachedInputTokens": 1006720, "cachedInputTokensReportedCalls": 42, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 10445, "reasoningOutputTokensReportedCalls": 42, "sourceReportedCostSum": 0.5072339999999999, "callsWithReportedCost": 42 }, "matchedIssues": 42, "callPercent": 0.03234600988863731, "tokenPercent": 0.03433533321949122, "sourceReportedCostPercent": 0.037724034559038964, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "proposals": 0, "matchedCalls": 0, "unmatchedProposals": 0, "matchedIssues": 0, "matchPercent": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "gemini3flash": { "trajectories": 498, "issues": 498, "proposals": 14, "matchedCalls": 0, "unmatchedProposals": 14, "matchedIssues": 0, "matchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSyntaxChecks": 14, "nextCommandDiffers": 14 } }, "gpt52": { "trajectories": 500, "issues": 500, "proposals": 367, "matchedCalls": 42, "unmatchedProposals": 325, "matchedIssues": 42, "matchPercent": 11.444141689373296, "matchedUsage": { "calls": 42, "callsWithUsage": 42, "inputTokens": 1101960, "inputTokensReportedCalls": 42, "outputTokens": 11742, "outputTokensReportedCalls": 42, "totalTokens": 1113702, "totalTokensReportedCalls": 42, "cachedInputTokens": 1006720, "cachedInputTokensReportedCalls": 42, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 10445, "reasoningOutputTokensReportedCalls": 42, "sourceReportedCostSum": 0.5072339999999999, "callsWithReportedCost": 42 }, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "callPercent": 0.23972602739726026, "tokenPercent": 0.33821215049551817, "matchesWithReasoningOrProse": 42, "diagnostics": { "recognizedSyntaxChecks": 370, "nextCommandDiffers": 323, "nextNotSingletonBashWithResult": 2 } }, "opus45": { "trajectories": 500, "issues": 500, "proposals": 0, "matchedCalls": 0, "unmatchedProposals": 0, "matchedIssues": 0, "matchPercent": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "sonnet45": { "trajectories": 500, "issues": 500, "proposals": 1, "matchedCalls": 0, "unmatchedProposals": 1, "matchedIssues": 0, "matchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSyntaxChecks": 1, "nextCommandDiffers": 1 } } }, "partitions": { "discovery": { "trajectories": 784, "issues": 157, "proposals": 118, "matchedCalls": 10, "unmatchedProposals": 108, "matchedIssues": 10, "matchPercent": 8.474576271186441, "matchedUsage": { "calls": 10, "callsWithUsage": 10, "inputTokens": 275040, "inputTokensReportedCalls": 10, "outputTokens": 2766, "outputTokensReportedCalls": 10, "totalTokens": 277806, "totalTokensReportedCalls": 10, "cachedInputTokens": 264576, "cachedInputTokensReportedCalls": 10, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 2397, "reasoningOutputTokensReportedCalls": 10, "sourceReportedCostSum": 0.10333679999999999, "callsWithReportedCost": 10 }, "denominator": { "calls": 39747, "callsWithUsage": 39747, "inputTokens": 963762231, "inputTokensReportedCalls": 39747, "outputTokens": 13087687, "outputTokensReportedCalls": 39747, "totalTokens": 976849918, "totalTokensReportedCalls": 39747, "cachedInputTokens": 748649310, "cachedInputTokensReportedCalls": 37079, "cacheCreationInputTokens": 9560939, "cacheCreationInputTokensReportedCalls": 12290, "reasoningOutputTokens": 7163382, "reasoningOutputTokensReportedCalls": 39747, "sourceReportedCostSum": 408.1896112699991, "callsWithReportedCost": 39747 }, "callPercent": 0.025159131506780386, "tokenPercent": 0.028438964356856302, "matchesWithReasoningOrProse": 10, "diagnostics": { "recognizedSyntaxChecks": 118, "nextCommandDiffers": 108 } }, "validation": { "trajectories": 1714, "issues": 343, "proposals": 264, "matchedCalls": 32, "unmatchedProposals": 232, "matchedIssues": 32, "matchPercent": 12.121212121212121, "matchedUsage": { "calls": 32, "callsWithUsage": 32, "inputTokens": 826920, "inputTokensReportedCalls": 32, "outputTokens": 8976, "outputTokensReportedCalls": 32, "totalTokens": 835896, "totalTokensReportedCalls": 32, "cachedInputTokens": 742144, "cachedInputTokensReportedCalls": 32, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 8048, "reasoningOutputTokensReportedCalls": 32, "sourceReportedCostSum": 0.40389719999999996, "callsWithReportedCost": 32 }, "denominator": { "calls": 90099, "callsWithUsage": 90099, "inputTokens": 2237463457, "inputTokensReportedCalls": 90099, "outputTokens": 29289921, "outputTokensReportedCalls": 90099, "totalTokens": 2266753378, "totalTokensReportedCalls": 90099, "cachedInputTokens": 1750216564, "cachedInputTokensReportedCalls": 84149, "cacheCreationInputTokens": 21986961, "cacheCreationInputTokensReportedCalls": 28299, "reasoningOutputTokens": 15912443, "reasoningOutputTokensReportedCalls": 90099, "sourceReportedCostSum": 936.4014589299964, "callsWithReportedCost": 90099 }, "callPercent": 0.03551648741939422, "tokenPercent": 0.036876354000960045, "matchesWithReasoningOrProse": 32, "diagnostics": { "recognizedSyntaxChecks": 267, "nextCommandDiffers": 230, "nextNotSingletonBashWithResult": 2 } } } }, { "program": "missing_rg_literal_search_repair_v3", "title": "Repair a missing ripgrep with a literal file search", "round": "new", "sourceReport": "candidate-command-repair-v3.json", "proposals": 11, "proposalMatchPercent": 81.81818181818181, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 35540, "inputTokensReportedCalls": 9, "outputTokens": 506, "outputTokensReportedCalls": 9, "totalTokens": 36046, "totalTokensReportedCalls": 9, "cachedInputTokens": 19456, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 116, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.0386358, "callsWithReportedCost": 9 }, "matchedIssues": 9, "callPercent": 0.006931287833279423, "tokenPercent": 0.0011112949615155404, "sourceReportedCostPercent": 0.002873423813104244, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": {} }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": {} }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 11, "matchedCalls": 9, "matchedIssues": 9, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 35540, "inputTokensReportedCalls": 9, "outputTokens": 506, "outputTokensReportedCalls": 9, "totalTokens": 36046, "totalTokensReportedCalls": 9, "cachedInputTokens": 19456, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 116, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.0386358, "callsWithReportedCost": 9 }, "matchPercent": 81.81818181818181, "callPercent": 0.05136986301369863, "tokenPercent": 0.01094655049264655, "diagnostics": { "nativeMissingRgErrors": 106, "nextCommandDiffers": 2 } }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": {} }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": {} } }, "partitions": { "discovery": { "trajectories": 820, "issues": 164, "denominator": { "calls": 42057, "callsWithUsage": 42057, "inputTokens": 1018758618, "inputTokensReportedCalls": 42057, "outputTokens": 13683908, "outputTokensReportedCalls": 42057, "totalTokens": 1032442526, "totalTokensReportedCalls": 42057, "cachedInputTokens": 789093892, "cachedInputTokensReportedCalls": 39053, "cacheCreationInputTokens": 10021340, "cacheCreationInputTokensReportedCalls": 12796, "reasoningOutputTokens": 7444696, "reasoningOutputTokensReportedCalls": 42057, "sourceReportedCostSum": 425.7461344200021, "callsWithReportedCost": 42057 }, "proposals": 1, "matchedCalls": 1, "matchedIssues": 1, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 5647, "inputTokensReportedCalls": 1, "outputTokens": 59, "outputTokensReportedCalls": 1, "totalTokens": 5706, "totalTokensReportedCalls": 1, "cachedInputTokens": 5504, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 12, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.00203945, "callsWithReportedCost": 1 }, "matchPercent": 100.0, "callPercent": 0.002377725467817486, "tokenPercent": 0.0005526699894963451, "diagnostics": { "nativeMissingRgErrors": 33 } }, "validation": { "trajectories": 1678, "issues": 336, "denominator": { "calls": 87789, "callsWithUsage": 87789, "inputTokens": 2182467070, "inputTokensReportedCalls": 87789, "outputTokens": 28693700, "outputTokensReportedCalls": 87789, "totalTokens": 2211160770, "totalTokensReportedCalls": 87789, "cachedInputTokens": 1709771982, "cachedInputTokensReportedCalls": 82175, "cacheCreationInputTokens": 21526560, "cacheCreationInputTokensReportedCalls": 27793, "reasoningOutputTokens": 15631129, "reasoningOutputTokensReportedCalls": 87789, "sourceReportedCostSum": 918.8449357799973, "callsWithReportedCost": 87789 }, "proposals": 10, "matchedCalls": 8, "matchedIssues": 8, "matchedUsage": { "calls": 8, "callsWithUsage": 8, "inputTokens": 29893, "inputTokensReportedCalls": 8, "outputTokens": 447, "outputTokensReportedCalls": 8, "totalTokens": 30340, "totalTokensReportedCalls": 8, "cachedInputTokens": 13952, "cachedInputTokensReportedCalls": 8, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 104, "reasoningOutputTokensReportedCalls": 8, "sourceReportedCostSum": 0.03659635, "callsWithReportedCost": 8 }, "matchPercent": 80.0, "callPercent": 0.009112759001697252, "tokenPercent": 0.0013721299876354083, "diagnostics": { "nativeMissingRgErrors": 73, "nextCommandDiffers": 2 } } } }, { "program": "edit_compile_v3", "title": "Check syntax after editing a Python source file", "round": "new", "sourceReport": "candidate-edit-compile-v3.json", "proposals": 1574, "proposalMatchPercent": 1.9059720457433291, "matchedUsage": { "calls": 30, "callsWithUsage": 30, "inputTokens": 947946, "inputTokensReportedCalls": 30, "outputTokens": 1403, "outputTokensReportedCalls": 30, "totalTokens": 949349, "totalTokensReportedCalls": 30, "cachedInputTokens": 815104, "cachedInputTokensReportedCalls": 30, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 372, "reasoningOutputTokensReportedCalls": 30, "sourceReportedCostSum": 0.3947587, "callsWithReportedCost": 30 }, "matchedIssues": 26, "callPercent": 0.023104292777598078, "tokenPercent": 0.029268344904283882, "sourceReportedCostPercent": 0.02935901544707433, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "proposals": 530, "matchedCalls": 0, "unmatchedProposals": 530, "proposalMatchPercent": 0.0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "nextCommandDiffers": 529, "nextNotSingletonOrMissingResult": 1 } }, "gemini3flash": { "trajectories": 498, "issues": 498, "proposals": 232, "matchedCalls": 0, "unmatchedProposals": 232, "proposalMatchPercent": 0.0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "nextCommandDiffers": 232 } }, "gpt52": { "trajectories": 500, "issues": 500, "proposals": 557, "matchedCalls": 30, "unmatchedProposals": 527, "proposalMatchPercent": 5.3859964093357275, "matchedIssues": 26, "matchedUsage": { "calls": 30, "callsWithUsage": 30, "inputTokens": 947946, "inputTokensReportedCalls": 30, "outputTokens": 1403, "outputTokensReportedCalls": 30, "totalTokens": 949349, "totalTokensReportedCalls": 30, "cachedInputTokens": 815104, "cachedInputTokensReportedCalls": 30, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 372, "reasoningOutputTokensReportedCalls": 30, "sourceReportedCostSum": 0.3947587, "callsWithReportedCost": 30 }, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "callPercent": 0.17123287671232876, "tokenPercent": 0.2883009699729099, "matchesWithReasoningOrProse": 9, "diagnostics": { "nextCommandDiffers": 525, "nextNotSingletonOrMissingResult": 2 } }, "opus45": { "trajectories": 500, "issues": 500, "proposals": 188, "matchedCalls": 0, "unmatchedProposals": 188, "proposalMatchPercent": 0.0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "nextCommandDiffers": 180, "nextNotSingletonOrMissingResult": 8 } }, "sonnet45": { "trajectories": 500, "issues": 500, "proposals": 67, "matchedCalls": 0, "unmatchedProposals": 67, "proposalMatchPercent": 0.0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "nextCommandDiffers": 67 } } }, "partitions": { "discovery": { "trajectories": 440, "issues": 88, "proposals": 205, "matchedCalls": 9, "unmatchedProposals": 196, "proposalMatchPercent": 4.390243902439025, "matchedIssues": 7, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 377913, "inputTokensReportedCalls": 9, "outputTokens": 429, "outputTokensReportedCalls": 9, "totalTokens": 378342, "totalTokensReportedCalls": 9, "cachedInputTokens": 332928, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 113, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.14299214999999998, "callsWithReportedCost": 9 }, "denominator": { "calls": 22248, "callsWithUsage": 22248, "inputTokens": 542472411, "inputTokensReportedCalls": 22248, "outputTokens": 7505645, "outputTokensReportedCalls": 22248, "totalTokens": 549978056, "totalTokensReportedCalls": 22248, "cachedInputTokens": 420449769, "cachedInputTokensReportedCalls": 20717, "cacheCreationInputTokens": 5377412, "cacheCreationInputTokensReportedCalls": 6760, "reasoningOutputTokens": 4096689, "reasoningOutputTokensReportedCalls": 22248, "sourceReportedCostSum": 227.55783245999973, "callsWithReportedCost": 22248 }, "callPercent": 0.040453074433656956, "tokenPercent": 0.06879219922912706, "matchesWithReasoningOrProse": 3, "diagnostics": { "nextCommandDiffers": 195, "nextNotSingletonOrMissingResult": 1 } }, "validation": { "trajectories": 2058, "issues": 412, "proposals": 1369, "matchedCalls": 21, "unmatchedProposals": 1348, "proposalMatchPercent": 1.5339663988312637, "matchedIssues": 19, "matchedUsage": { "calls": 21, "callsWithUsage": 21, "inputTokens": 570033, "inputTokensReportedCalls": 21, "outputTokens": 974, "outputTokensReportedCalls": 21, "totalTokens": 571007, "totalTokensReportedCalls": 21, "cachedInputTokens": 482176, "cachedInputTokensReportedCalls": 21, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 259, "reasoningOutputTokensReportedCalls": 21, "sourceReportedCostSum": 0.25176655000000003, "callsWithReportedCost": 21 }, "denominator": { "calls": 107598, "callsWithUsage": 107598, "inputTokens": 2658753277, "inputTokensReportedCalls": 107598, "outputTokens": 34871963, "outputTokensReportedCalls": 107598, "totalTokens": 2693625240, "totalTokensReportedCalls": 107598, "cachedInputTokens": 2078416105, "cachedInputTokensReportedCalls": 100511, "cacheCreationInputTokens": 26170488, "cacheCreationInputTokensReportedCalls": 33829, "reasoningOutputTokens": 18979136, "reasoningOutputTokensReportedCalls": 107598, "sourceReportedCostSum": 1117.0332377399923, "callsWithReportedCost": 107598 }, "callPercent": 0.01951709139575085, "tokenPercent": 0.021198457436491798, "matchesWithReasoningOrProse": 6, "diagnostics": { "nextCommandDiffers": 1338, "nextNotSingletonOrMissingResult": 10 } } } } ], "previousUnion": { "matchedUsage": { "calls": 459, "callsWithUsage": 459, "inputTokens": 7442341, "inputTokensReportedCalls": 459, "outputTokens": 34131, "outputTokensReportedCalls": 459, "totalTokens": 7476472, "totalTokensReportedCalls": 459, "cachedInputTokens": 6075399, "cachedInputTokensReportedCalls": 452, "cacheCreationInputTokens": 311012, "cacheCreationInputTokensReportedCalls": 344, "reasoningOutputTokens": 4382, "reasoningOutputTokensReportedCalls": 459, "sourceReportedCostSum": 3.5179327900000033, "callsWithReportedCost": 459 }, "matchedIssues": 179, "callPercent": 0.3534956794972506, "tokenPercent": 0.23049896419885743, "sourceReportedCostPercent": 0.26163588826130846 }, "newRoundUnion": { "matchedUsage": { "calls": 112, "callsWithUsage": 112, "inputTokens": 2415757, "inputTokensReportedCalls": 112, "outputTokens": 16603, "outputTokensReportedCalls": 112, "totalTokens": 2432360, "totalTokensReportedCalls": 112, "cachedInputTokens": 2054772, "cachedInputTokensReportedCalls": 107, "cacheCreationInputTokens": 878, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 12461, "reasoningOutputTokensReportedCalls": 112, "sourceReportedCostSum": 1.0333790200000001, "callsWithReportedCost": 112 }, "matchedIssues": 101, "callPercent": 0.08625602636969949, "tokenPercent": 0.07498944161881872, "sourceReportedCostPercent": 0.07685452052320199, "duplicateTargetsRemoved": 0, "dependentChainsExcluded": 0 }, "combinedUnion": { "matchedUsage": { "calls": 571, "callsWithUsage": 571, "inputTokens": 9858098, "inputTokensReportedCalls": 571, "outputTokens": 50734, "outputTokensReportedCalls": 571, "totalTokens": 9908832, "totalTokensReportedCalls": 571, "cachedInputTokens": 8130171, "cachedInputTokensReportedCalls": 559, "cacheCreationInputTokens": 311890, "cacheCreationInputTokensReportedCalls": 348, "reasoningOutputTokens": 16843, "reasoningOutputTokensReportedCalls": 571, "sourceReportedCostSum": 4.55131181, "callsWithReportedCost": 571 }, "matchedIssues": 240, "callPercent": 0.43975170586695006, "tokenPercent": 0.30548840581767617, "sourceReportedCostPercent": 0.33849040878451014, "duplicateTargetsRemoved": 0, "dependentChainsExcluded": 0 }, "incrementalExactCalls": 112, "sourceManifestSha256": "73cccd8079f5c3150015207a7d102f1a4409c7996f0ea9ae1305f1b45239247b", "verifiedNewEvents": 112, "limitations": [ "Percentages divide matched original calls/tokens by the full corpus denominator, not by proposals.", "Proposal match percentage is descriptive precision, not deployment safety or task success.", "Programs see prior/current observations; future calls are replay labels only.", "Original recorded token counts are known; avoiding those calls is conditional on dispensable reasoning and successful program execution.", "Gross estimates exclude routing/program/false-positive costs and future context/cache effects.", "Per-program savings may overlap; combined totals deduplicate targets and exclude dependent adjacent substitutions.", "New candidates use candidate-specific discovery/validation issue partitions; previously inspected discovery issues are excluded from validation. This corpus is not untouched external validation.", "Coverage-only or EOF-equivalent scenarios are not added to exact totals.", "No recorded commands are executed and no paid model calls occur." ] }