{ "version": 1, "kind": "conditional-gross-program-replay-percentages", "trajectories": 2498, "uniqueIssues": 500, "denominator": { "calls": 129846, "callsWithUsage": 129846, "inputTokens": 3201225688, "inputTokensReportedCalls": 129846, "outputTokens": 42377608, "outputTokensReportedCalls": 129846, "totalTokens": 3243603296, "totalTokensReportedCalls": 129846, "cachedInputTokens": 2498865874, "cachedInputTokensReportedCalls": 121228, "cacheCreationInputTokens": 31547900, "cacheCreationInputTokensReportedCalls": 40589, "reasoningOutputTokens": 23075825, "reasoningOutputTokensReportedCalls": 129846, "sourceReportedCostSum": 1344.5910701999999, "callsWithReportedCost": 129846 }, "programs": [ { "program": "bounded_read_continuation_v4", "title": "Continue a bounded source read", "round": "previous", "proposals": 12335, "proposalMatchPercent": 0.3080664775030401, "matchedUsage": { "calls": 38, "callsWithUsage": 38, "inputTokens": 515742, "inputTokensReportedCalls": 38, "outputTokens": 2592, "outputTokensReportedCalls": 38, "totalTokens": 518334, "totalTokensReportedCalls": 38, "cachedInputTokens": 356098, "cachedInputTokensReportedCalls": 36, "cacheCreationInputTokens": 10620, "cacheCreationInputTokensReportedCalls": 9, "reasoningOutputTokens": 576, "reasoningOutputTokensReportedCalls": 38, "sourceReportedCostSum": 0.30480796, "callsWithReportedCost": 38 }, "matchedIssues": 36, "callPercent": 0.029265437518290898, "tokenPercent": 0.015980190938861348, "sourceReportedCostPercent": 0.022669194133102612 }, { "program": "edit_compile_v3", "title": "Check syntax after a source edit", "round": "previous", "proposals": 1574, "proposalMatchPercent": 1.9059720457433291, "matchedUsage": { "calls": 30, "callsWithUsage": 30, "inputTokens": 947946, "inputTokensReportedCalls": 30, "outputTokens": 1403, "outputTokensReportedCalls": 30, "totalTokens": 949349, "totalTokensReportedCalls": 30, "cachedInputTokens": 815104, "cachedInputTokensReportedCalls": 30, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 372, "reasoningOutputTokensReportedCalls": 30, "sourceReportedCostSum": 0.3947587, "callsWithReportedCost": 30 }, "matchedIssues": 26, "callPercent": 0.023104292777598078, "tokenPercent": 0.029268344904283882, "sourceReportedCostPercent": 0.02935901544707433 }, { "program": "edited_file_patch_review_v4", "title": "Review the edited file diff", "round": "previous", "proposals": 1574, "proposalMatchPercent": 0.25412960609911056, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 121735, "inputTokensReportedCalls": 4, "outputTokens": 225, "outputTokensReportedCalls": 4, "totalTokens": 121960, "totalTokensReportedCalls": 4, "cachedInputTokens": 110488, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 267, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 65, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.0251073, "callsWithReportedCost": 4 }, "matchedIssues": 4, "callPercent": 0.0030805723703464104, "tokenPercent": 0.003760015910404353, "sourceReportedCostPercent": 0.0018672814773539614 }, { "program": "inspect_unique_small_file_v3", "title": "Read a uniquely identified small file", "round": "previous", "proposals": 4191, "proposalMatchPercent": 0.7396802672393223, "matchedUsage": { "calls": 31, "callsWithUsage": 31, "inputTokens": 330311, "inputTokensReportedCalls": 31, "outputTokens": 2952, "outputTokensReportedCalls": 31, "totalTokens": 333263, "totalTokensReportedCalls": 31, "cachedInputTokens": 213492, "cachedInputTokensReportedCalls": 26, "cacheCreationInputTokens": 878, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 1528, "reasoningOutputTokensReportedCalls": 31, "sourceReportedCostSum": 0.09275051999999999, "callsWithReportedCost": 31 }, "matchedIssues": 30, "callPercent": 0.02387443587018468, "tokenPercent": 0.01027446853352809, "sourceReportedCostPercent": 0.006898046703984424 }, { "program": "missing_rg_literal_search_repair_v3", "title": "Repair a missing ripgrep command", "round": "previous", "proposals": 11, "proposalMatchPercent": 81.81818181818181, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 35540, "inputTokensReportedCalls": 9, "outputTokens": 506, "outputTokensReportedCalls": 9, "totalTokens": 36046, "totalTokensReportedCalls": 9, "cachedInputTokens": 19456, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 116, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.0386358, "callsWithReportedCost": 9 }, "matchedIssues": 9, "callPercent": 0.006931287833279423, "tokenPercent": 0.0011112949615155404, "sourceReportedCostPercent": 0.002873423813104244 }, { "program": "registered_failed_test_rerun_v2", "title": "Rerun a failed assertion regression", "round": "previous", "proposals": 3, "proposalMatchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0 }, { "program": "registered_test_rerun_v2", "title": "Rerun a previously passing test", "round": "previous", "proposals": 92, "proposalMatchPercent": 19.565217391304348, "matchedUsage": { "calls": 18, "callsWithUsage": 18, "inputTokens": 444361, "inputTokensReportedCalls": 18, "outputTokens": 679, "outputTokensReportedCalls": 18, "totalTokens": 445040, "totalTokensReportedCalls": 18, "cachedInputTokens": 262107, "cachedInputTokensReportedCalls": 11, "cacheCreationInputTokens": 138, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 254, "reasoningOutputTokensReportedCalls": 18, "sourceReportedCostSum": 0.11198141, "callsWithReportedCost": 18 }, "matchedIssues": 17, "callPercent": 0.013862575666558847, "tokenPercent": 0.013720543463154749, "sourceReportedCostPercent": 0.008328287498097353 }, { "program": "search_read_v2", "title": "Read context around the only search hit", "round": "previous", "proposals": 3572, "proposalMatchPercent": 0.083986562150056, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 13897, "inputTokensReportedCalls": 3, "outputTokens": 231, "outputTokensReportedCalls": 3, "totalTokens": 14128, "totalTokensReportedCalls": 3, "cachedInputTokens": 957, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 66, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.00359573, "callsWithReportedCost": 3 }, "matchedIssues": 3, "callPercent": 0.002310429277759808, "tokenPercent": 0.00043556497853552555, "sourceReportedCostPercent": 0.00026742182658294445 }, { "program": "syntax_check_status_v3", "title": "Inspect status after a syntax check", "round": "previous", "proposals": 382, "proposalMatchPercent": 10.99476439790576, "matchedUsage": { "calls": 42, "callsWithUsage": 42, "inputTokens": 1101960, "inputTokensReportedCalls": 42, "outputTokens": 11742, "outputTokensReportedCalls": 42, "totalTokens": 1113702, "totalTokensReportedCalls": 42, "cachedInputTokens": 1006720, "cachedInputTokensReportedCalls": 42, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 10445, "reasoningOutputTokensReportedCalls": 42, "sourceReportedCostSum": 0.5072339999999999, "callsWithReportedCost": 42 }, "matchedIssues": 42, "callPercent": 0.03234600988863731, "tokenPercent": 0.03433533321949122, "sourceReportedCostPercent": 0.037724034559038964 }, { "program": "traceback_context_v4", "title": "Read source context from a traceback", "round": "previous", "proposals": 862, "proposalMatchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0 }, { "program": "write_then_run_v2", "title": "Write a script and run it", "round": "previous", "proposals": 2238, "proposalMatchPercent": 19.571045576407506, "matchedUsage": { "calls": 438, "callsWithUsage": 438, "inputTokens": 6984083, "inputTokensReportedCalls": 438, "outputTokens": 33221, "outputTokensReportedCalls": 438, "totalTokens": 7017304, "totalTokensReportedCalls": 438, "cachedInputTokens": 5812335, "cachedInputTokensReportedCalls": 438, "cacheCreationInputTokens": 310874, "cacheCreationInputTokensReportedCalls": 343, "reasoningOutputTokens": 4062, "reasoningOutputTokensReportedCalls": 438, "sourceReportedCostSum": 3.402355650000003, "callsWithReportedCost": 438 }, "matchedIssues": 166, "callPercent": 0.3373226745529319, "tokenPercent": 0.21634285575716716, "sourceReportedCostPercent": 0.2530401789366281 }, { "program": "missing_pytest_module_retry_v5", "title": "Retry a missing pytest command through registered Python", "round": "new", "sourceReport": "candidate-pytest-repair-v5.json", "proposals": 106, "proposalMatchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "retryExitCodes": {}, "diagnostics": {} }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 95, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "retryExitCodes": {}, "diagnostics": { "nativeMissingPytestErrors": 110, "nextCommandDiffers": 83, "nextRetryUnavailableOrUnbounded": 12, "matchingCommandButPytestModuleMissing": 12 } }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 11, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "retryExitCodes": {}, "diagnostics": { "nativeMissingPytestErrors": 12, "nextCommandDiffers": 11 } }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "retryExitCodes": {}, "diagnostics": {} }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "retryExitCodes": {}, "diagnostics": {} } }, "partitions": { "discovery": { "trajectories": 550, "issues": 110, "denominator": { "calls": 28862, "callsWithUsage": 28862, "inputTokens": 702102702, "inputTokensReportedCalls": 28862, "outputTokens": 9426963, "outputTokensReportedCalls": 28862, "totalTokens": 711529665, "totalTokensReportedCalls": 28862, "cachedInputTokens": 543584191, "cachedInputTokensReportedCalls": 27013, "cacheCreationInputTokens": 6709415, "cacheCreationInputTokensReportedCalls": 8757, "reasoningOutputTokens": 5220878, "reasoningOutputTokensReportedCalls": 28862, "sourceReportedCostSum": 291.47569229000186, "callsWithReportedCost": 28862 }, "proposals": 21, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "retryExitCodes": {}, "diagnostics": { "nativeMissingPytestErrors": 25, "nextCommandDiffers": 17, "nextRetryUnavailableOrUnbounded": 4, "matchingCommandButPytestModuleMissing": 4 } }, "validation": { "trajectories": 1948, "issues": 390, "denominator": { "calls": 100984, "callsWithUsage": 100984, "inputTokens": 2499122986, "inputTokensReportedCalls": 100984, "outputTokens": 32950645, "outputTokensReportedCalls": 100984, "totalTokens": 2532073631, "totalTokensReportedCalls": 100984, "cachedInputTokens": 1955281683, "cachedInputTokensReportedCalls": 94215, "cacheCreationInputTokens": 24838485, "cacheCreationInputTokensReportedCalls": 31832, "reasoningOutputTokens": 17854947, "reasoningOutputTokensReportedCalls": 100984, "sourceReportedCostSum": 1053.1153779099996, "callsWithReportedCost": 100984 }, "proposals": 85, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "retryExitCodes": {}, "diagnostics": { "nativeMissingPytestErrors": 97, "nextCommandDiffers": 77, "nextRetryUnavailableOrUnbounded": 8, "matchingCommandButPytestModuleMissing": 8 } } } }, { "program": "known_path_read_repair_v5", "title": "Repair a read using a uniquely known file path", "round": "new", "sourceReport": "candidate-path-repair-v5.json", "proposals": 0, "proposalMatchPercent": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": { "supportedNativeMissingReadErrors": 23 } }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": { "supportedNativeMissingReadErrors": 5 } }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": {} }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": { "supportedNativeMissingReadErrors": 1 } }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": {} } }, "partitions": { "discovery": { "trajectories": 550, "issues": 110, "denominator": { "calls": 28862, "callsWithUsage": 28862, "inputTokens": 702102702, "inputTokensReportedCalls": 28862, "outputTokens": 9426963, "outputTokensReportedCalls": 28862, "totalTokens": 711529665, "totalTokensReportedCalls": 28862, "cachedInputTokens": 543584191, "cachedInputTokensReportedCalls": 27013, "cacheCreationInputTokens": 6709415, "cacheCreationInputTokensReportedCalls": 8757, "reasoningOutputTokens": 5220878, "reasoningOutputTokensReportedCalls": 28862, "sourceReportedCostSum": 291.47569229000186, "callsWithReportedCost": 28862 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": { "supportedNativeMissingReadErrors": 8 } }, "validation": { "trajectories": 1948, "issues": 390, "denominator": { "calls": 100984, "callsWithUsage": 100984, "inputTokens": 2499122986, "inputTokensReportedCalls": 100984, "outputTokens": 32950645, "outputTokensReportedCalls": 100984, "totalTokens": 2532073631, "totalTokensReportedCalls": 100984, "cachedInputTokens": 1955281683, "cachedInputTokensReportedCalls": 94215, "cacheCreationInputTokens": 24838485, "cacheCreationInputTokensReportedCalls": 31832, "reasoningOutputTokens": 17854947, "reasoningOutputTokensReportedCalls": 100984, "sourceReportedCostSum": 1053.1153779099996, "callsWithReportedCost": 100984 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0, "callPercent": 0.0, "tokenPercent": 0.0, "diagnostics": { "supportedNativeMissingReadErrors": 21 } } } }, { "program": "empty_search_parent_scope_v5", "title": "Broaden an empty file search to its parent directory", "round": "new", "sourceReport": "candidate-empty-search-v5.json", "proposals": 784, "proposalMatchPercent": 1.7857142857142858, "matchedUsage": { "calls": 14, "callsWithUsage": 14, "inputTokens": 395884, "inputTokensReportedCalls": 14, "outputTokens": 805, "outputTokensReportedCalls": 14, "totalTokens": 396689, "totalTokensReportedCalls": 14, "cachedInputTokens": 267426, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 395, "reasoningOutputTokensReportedCalls": 14, "sourceReportedCostSum": 0.07653624, "callsWithReportedCost": 14 }, "matchedIssues": 13, "callPercent": 0.010782003296212437, "tokenPercent": 0.012229886450331193, "sourceReportedCostPercent": 0.005692157392404495, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 370, "matchedCalls": 1, "matchedIssues": 1, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 13900, "inputTokensReportedCalls": 1, "outputTokens": 96, "outputTokensReportedCalls": 1, "totalTokens": 13996, "totalTokensReportedCalls": 1, "cachedInputTokens": 319, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 37, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.00361539, "callsWithReportedCost": 1 }, "matchPercent": 0.2702702702702703, "callPercent": 0.0022892200627246295, "tokenPercent": 0.0011749767334080792, "matchesWithReasoningOrProse": 1, "diagnostics": { "emptyGrepLikeResponses": 1260, "nextCommandDiffers": 369 } }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 317, "matchedCalls": 13, "matchedIssues": 13, "matchedUsage": { "calls": 13, "callsWithUsage": 13, "inputTokens": 381984, "inputTokensReportedCalls": 13, "outputTokens": 709, "outputTokensReportedCalls": 13, "totalTokens": 382693, "totalTokensReportedCalls": 13, "cachedInputTokens": 267107, "cachedInputTokensReportedCalls": 8, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 358, "reasoningOutputTokensReportedCalls": 13, "sourceReportedCostSum": 0.07292085000000001, "callsWithReportedCost": 13 }, "matchPercent": 4.100946372239748, "callPercent": 0.04633920296570899, "tokenPercent": 0.044603002543040025, "matchesWithReasoningOrProse": 13, "diagnostics": { "emptyGrepLikeResponses": 936, "nextCommandDiffers": 303, "nextNotSingletonObservedBash": 1 } }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 31, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "emptyGrepLikeResponses": 99, "nextCommandDiffers": 31 } }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 11, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "emptyGrepLikeResponses": 76, "nextCommandDiffers": 9, "nextNotSingletonObservedBash": 2 } }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 55, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": 0.0, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "emptyGrepLikeResponses": 328, "nextCommandDiffers": 54, "nextNotSingletonObservedBash": 1 } } }, "partitions": { "discovery": { "trajectories": 550, "issues": 110, "denominator": { "calls": 28862, "callsWithUsage": 28862, "inputTokens": 702102702, "inputTokensReportedCalls": 28862, "outputTokens": 9426963, "outputTokensReportedCalls": 28862, "totalTokens": 711529665, "totalTokensReportedCalls": 28862, "cachedInputTokens": 543584191, "cachedInputTokensReportedCalls": 27013, "cacheCreationInputTokens": 6709415, "cacheCreationInputTokensReportedCalls": 8757, "reasoningOutputTokens": 5220878, "reasoningOutputTokensReportedCalls": 28862, "sourceReportedCostSum": 291.47569229000186, "callsWithReportedCost": 28862 }, "proposals": 153, "matchedCalls": 5, "matchedIssues": 4, "matchedUsage": { "calls": 5, "callsWithUsage": 5, "inputTokens": 115545, "inputTokensReportedCalls": 5, "outputTokens": 293, "outputTokensReportedCalls": 5, "totalTokens": 115838, "totalTokensReportedCalls": 5, "cachedInputTokens": 71180, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 121, "reasoningOutputTokensReportedCalls": 5, "sourceReportedCostSum": 0.02314144, "callsWithReportedCost": 5 }, "matchPercent": 3.2679738562091503, "callPercent": 0.0173238167833137, "tokenPercent": 0.016280136401621428, "matchesWithReasoningOrProse": 5, "diagnostics": { "emptyGrepLikeResponses": 568, "nextCommandDiffers": 147, "nextNotSingletonObservedBash": 1 } }, "validation": { "trajectories": 1948, "issues": 390, "denominator": { "calls": 100984, "callsWithUsage": 100984, "inputTokens": 2499122986, "inputTokensReportedCalls": 100984, "outputTokens": 32950645, "outputTokensReportedCalls": 100984, "totalTokens": 2532073631, "totalTokensReportedCalls": 100984, "cachedInputTokens": 1955281683, "cachedInputTokensReportedCalls": 94215, "cacheCreationInputTokens": 24838485, "cacheCreationInputTokensReportedCalls": 31832, "reasoningOutputTokens": 17854947, "reasoningOutputTokensReportedCalls": 100984, "sourceReportedCostSum": 1053.1153779099996, "callsWithReportedCost": 100984 }, "proposals": 631, "matchedCalls": 9, "matchedIssues": 9, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 280339, "inputTokensReportedCalls": 9, "outputTokens": 512, "outputTokensReportedCalls": 9, "totalTokens": 280851, "totalTokensReportedCalls": 9, "cachedInputTokens": 196246, "cachedInputTokensReportedCalls": 6, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 274, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.05339479999999999, "callsWithReportedCost": 9 }, "matchPercent": 1.4263074484944533, "callPercent": 0.008912302939079459, "tokenPercent": 0.011091739061675019, "matchesWithReasoningOrProse": 9, "diagnostics": { "emptyGrepLikeResponses": 2131, "nextCommandDiffers": 619, "nextNotSingletonObservedBash": 3 } } } }, { "program": "explicit_finish_after_verification_v5", "title": "Finish only under an explicit registered verification policy", "round": "new", "sourceReport": "candidate-finalize-v5.json", "proposals": 0, "proposalMatchPercent": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchedIssues": 0, "callPercent": 0.0, "tokenPercent": 0.0, "sourceReportedCostPercent": 0.0, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "diffCheckLikeTurns": 37, "supportedPassedFullTreeCheckTurns": 7 } }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "diffCheckLikeTurns": 3 } } }, "partitions": { "discovery": { "trajectories": 550, "issues": 110, "denominator": { "calls": 28862, "callsWithUsage": 28862, "inputTokens": 702102702, "inputTokensReportedCalls": 28862, "outputTokens": 9426963, "outputTokensReportedCalls": 28862, "totalTokens": 711529665, "totalTokensReportedCalls": 28862, "cachedInputTokens": 543584191, "cachedInputTokensReportedCalls": 27013, "cacheCreationInputTokens": 6709415, "cacheCreationInputTokensReportedCalls": 8757, "reasoningOutputTokens": 5220878, "reasoningOutputTokensReportedCalls": 28862, "sourceReportedCostSum": 291.47569229000186, "callsWithReportedCost": 28862 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "diffCheckLikeTurns": 13, "supportedPassedFullTreeCheckTurns": 2 } }, "validation": { "trajectories": 1948, "issues": 390, "denominator": { "calls": 100984, "callsWithUsage": 100984, "inputTokens": 2499122986, "inputTokensReportedCalls": 100984, "outputTokens": 32950645, "outputTokensReportedCalls": 100984, "totalTokens": 2532073631, "totalTokensReportedCalls": 100984, "cachedInputTokens": 1955281683, "cachedInputTokensReportedCalls": 94215, "cacheCreationInputTokens": 24838485, "cacheCreationInputTokensReportedCalls": 31832, "reasoningOutputTokens": 17854947, "reasoningOutputTokensReportedCalls": 100984, "sourceReportedCostSum": 1053.1153779099996, "callsWithReportedCost": 100984 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "diffCheckLikeTurns": 27, "supportedPassedFullTreeCheckTurns": 5 } } } } ], "previousUnion": { "matchedUsage": { "calls": 613, "callsWithUsage": 613, "inputTokens": 10495575, "inputTokensReportedCalls": 613, "outputTokens": 53551, "outputTokensReportedCalls": 613, "totalTokens": 10549126, "totalTokensReportedCalls": 613, "cachedInputTokens": 8596757, "cachedInputTokensReportedCalls": 599, "cacheCreationInputTokens": 322777, "cacheCreationInputTokensReportedCalls": 358, "reasoningOutputTokens": 17484, "reasoningOutputTokensReportedCalls": 613, "sourceReportedCostSum": 4.881227070000003, "callsWithReportedCost": 613 }, "matchedIssues": 256, "callPercent": 0.4720977157555874, "tokenPercent": 0.32522861266694186, "sourceReportedCostPercent": 0.363026884394967 }, "newRoundUnion": { "matchedUsage": { "calls": 14, "callsWithUsage": 14, "inputTokens": 395884, "inputTokensReportedCalls": 14, "outputTokens": 805, "outputTokensReportedCalls": 14, "totalTokens": 396689, "totalTokensReportedCalls": 14, "cachedInputTokens": 267426, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 395, "reasoningOutputTokensReportedCalls": 14, "sourceReportedCostSum": 0.07653624, "callsWithReportedCost": 14 }, "matchedIssues": 13, "callPercent": 0.010782003296212437, "tokenPercent": 0.012229886450331193, "sourceReportedCostPercent": 0.005692157392404495, "duplicateTargetsRemoved": 0, "dependentChainsExcluded": 0 }, "combinedUnion": { "matchedUsage": { "calls": 627, "callsWithUsage": 627, "inputTokens": 10891459, "inputTokensReportedCalls": 627, "outputTokens": 54356, "outputTokensReportedCalls": 627, "totalTokens": 10945815, "totalTokensReportedCalls": 627, "cachedInputTokens": 8864183, "cachedInputTokensReportedCalls": 608, "cacheCreationInputTokens": 322777, "cacheCreationInputTokensReportedCalls": 358, "reasoningOutputTokens": 17879, "reasoningOutputTokensReportedCalls": 627, "sourceReportedCostSum": 4.957763310000003, "callsWithReportedCost": 627 }, "matchedIssues": 264, "callPercent": 0.4828797190517998, "tokenPercent": 0.33745849911727305, "sourceReportedCostPercent": 0.36871904178737147, "duplicateTargetsRemoved": 0, "dependentChainsExcluded": 0 }, "incrementalExactCalls": 14, "sourceManifestSha256": "73cccd8079f5c3150015207a7d102f1a4409c7996f0ea9ae1305f1b45239247b", "verifiedNewEvents": 14, "limitations": [ "Percentages divide matched original calls/tokens by the full corpus denominator, not by proposals.", "Proposal match percentage is descriptive precision, not deployment safety or task success.", "Programs see prior/current observations; future calls are replay labels only.", "Original recorded token counts are known; avoiding those calls is conditional on dispensable reasoning and successful program execution.", "Gross estimates exclude routing/program/false-positive costs and future context/cache effects.", "Per-program savings may overlap; combined totals deduplicate targets and exclude dependent adjacent substitutions.", "New candidates use candidate-specific discovery/validation issue partitions; previously inspected discovery issues are excluded from validation. This corpus is not untouched external validation.", "Coverage-only or EOF-equivalent scenarios are not added to exact totals.", "No recorded commands are executed and no paid model calls occur." ] }