{ "version": 1, "kind": "offline-program-classifier-batch-replay", "classifier": "Deterministic eligibility guards and argument planners; no model API or learned router invoked.", "status": "complete", "requestedTrajectories": 2498, "failedTrajectories": 0, "errorsByType": {}, "sources": [ { "label": "sonnet45", "submission": "20260217_mini-v2.0.0_claude-4-5-sonnet-high", "githubRevision": "40f164d5b8f1d249bf95a6df8b74b577fd8e519d", "metadataURL": "https://raw.githubusercontent.com/SWE-bench/experiments/40f164d5b8f1d249bf95a6df8b74b577fd8e519d/evaluation/verified/20260217_mini-v2.0.0_claude-4-5-sonnet-high/metadata.yaml", "metadataSha256": "a158a82473a28ec37c2639d5d309dbba23cd4f75d22763c9b5e67b4b5830d308", "trajectoryPrefix": "https://swe-bench-submissions.s3.amazonaws.com/bash-only/20260217_mini-v2.0.0_claude-4-5-sonnet-high/trajs/", "availableTrajectories": 500, "availableBytes": 298722547 }, { "label": "gpt52", "submission": "20260217_mini-v2.0.0_gpt-5-2-high", "githubRevision": "40f164d5b8f1d249bf95a6df8b74b577fd8e519d", "metadataURL": "https://raw.githubusercontent.com/SWE-bench/experiments/40f164d5b8f1d249bf95a6df8b74b577fd8e519d/evaluation/verified/20260217_mini-v2.0.0_gpt-5-2-high/metadata.yaml", "metadataSha256": "969a7c91b351fd0ac9ce47fddec706a9babbc15a84bc2022dd6f138461858d75", "trajectoryPrefix": "https://swe-bench-submissions.s3.amazonaws.com/bash-only/20260217_mini-v2.0.0_gpt-5-2-high/trajs/", "availableTrajectories": 500, "availableBytes": 129942497 }, { "label": "deepseek32", "submission": "20260217_mini-v2.0.0_deepseek-3-2-high", "githubRevision": "40f164d5b8f1d249bf95a6df8b74b577fd8e519d", "metadataURL": "https://raw.githubusercontent.com/SWE-bench/experiments/40f164d5b8f1d249bf95a6df8b74b577fd8e519d/evaluation/verified/20260217_mini-v2.0.0_deepseek-3-2-high/metadata.yaml", "metadataSha256": "cc7f67a07a5059e6cd884ef5d000d25b6284845d4bb09253636a885af58cff43", "trajectoryPrefix": "https://swe-bench-submissions.s3.amazonaws.com/bash-only/20260217_mini-v2.0.0_deepseek-3-2-high/trajs/", "availableTrajectories": 500, "availableBytes": 392738581 }, { "label": "gemini3flash", "submission": "20260217_mini-v2.0.0_gemini-3-flash-high", "githubRevision": "40f164d5b8f1d249bf95a6df8b74b577fd8e519d", "metadataURL": "https://raw.githubusercontent.com/SWE-bench/experiments/40f164d5b8f1d249bf95a6df8b74b577fd8e519d/evaluation/verified/20260217_mini-v2.0.0_gemini-3-flash-high/metadata.yaml", "metadataSha256": "1f0567e39e292e9216cb669def83e6f6a9bd02ee3d9731c4f23cad13ac42816b", "trajectoryPrefix": "https://swe-bench-submissions.s3.amazonaws.com/bash-only/20260217_mini-v2.0.0_gemini-3-flash-high/trajs/", "availableTrajectories": 500, "availableBytes": 625003330 }, { "label": "opus45", "submission": "20260217_mini-v2.0.0_claude-4-5-opus-high", "githubRevision": "40f164d5b8f1d249bf95a6df8b74b577fd8e519d", "metadataURL": "https://raw.githubusercontent.com/SWE-bench/experiments/40f164d5b8f1d249bf95a6df8b74b577fd8e519d/evaluation/verified/20260217_mini-v2.0.0_claude-4-5-opus-high/metadata.yaml", "metadataSha256": "65f42343f076d47de2b7fe175469bea9123ea61d033c67bb9c3bbc5f9a508f3f", "trajectoryPrefix": "https://swe-bench-submissions.s3.amazonaws.com/bash-only/20260217_mini-v2.0.0_claude-4-5-opus-high/trajs/", "availableTrajectories": 500, "availableBytes": 221052108 } ], "sourceManifestSha256": "73cccd8079f5c3150015207a7d102f1a4409c7996f0ea9ae1305f1b45239247b", "policyHash": "2d777d1139a763e7fde7ca0953d8b1655fe56748af70fdd0d74eba6383c98bbe", "policySources": { "batch-program-replay.py": "056dfc1dc87d478ca93cfddb09e0dbd21b79cf75297a659c0338913de06d305a", "trace_batch_core.py": "f78906c72a141f8a4305ea82c6f436e8f21710a99077ab93e9dec4516b45e1f4", "estimate-recorded-usage.py": "2875b21ccf205f09ec737713ed1516cb0d19c9885b20d2b1f8a2945681b0a6c0", "mine-public-mini.py": "c7b4b31df1be3ecd64c37d3e9884da054e8680ebaeb6c2c25ec281a3c2405e38", "candidate-write-run-v2.py": "a38f7a197ce7d91bd228a05be9a36cb3e68183d37632dcfb82876953a7a49a4b", "candidate-search-read-v2.py": "197c7052f831c631597f6e6cb815863c11ff51827f178563094144afe13214fe", "candidate-rerun-v2.py": "a19ae88918d77afb5db77bf1f0a703250500ada73107a37310bbb21a813eb44e", "candidate-failed-rerun-v2.py": "66b47d07a5fb4cb8839853363f1fb7d42b9b75ed9b85b11e082bd7d26ac0d635", "aggregate-candidate-estimates.py": "254a9fd93c9d373d10dfa942f93e55fe9209d62cda33398a2315d709ffaab810", "candidate-inspect-file-v3.py": "782082f9fe6fd407714e3fb7188916ded28dce510567d7b36b3e4e092912df28", "candidate-diff-check-v3.py": "fdce9109b7df03d6043f090608248e29100adcf7132b3261f9a232b462e08cb5", "candidate-command-repair-v3.py": "fcaf800439198c64ea488b1564d0325437a36aab0fa28b54972448c69cc83af6", "candidate-edit-compile-v3.py": "33293cc20631fd2d27a035fc7689367caf7f69c682eb7bfc91b28911d24fbc83" }, "performance": { "workers": 4, "wallSeconds": 29.359322208000002, "cacheHits": 0, "classifiedTrajectories": 2498, "trajectoriesPerSecond": 85.08370807413702, "responsesPerSecond": 4422.64978326437, "workerSecondsNewWork": 78.6022450389999 }, "reportedApiCalls": 130435, "trajectoriesWithReportedApiCalls": 2498, "unrecordedResponseCount": 589, "trajectoriesWithCounterMismatch": 296, "trajectories": 2498, "uniqueIssues": 500, "denominator": { "calls": 129846, "callsWithUsage": 129846, "inputTokens": 3201225688, "inputTokensReportedCalls": 129846, "outputTokens": 42377608, "outputTokensReportedCalls": 129846, "totalTokens": 3243603296, "totalTokensReportedCalls": 129846, "cachedInputTokens": 2498865874, "cachedInputTokensReportedCalls": 121228, "cacheCreationInputTokens": 31547900, "cacheCreationInputTokensReportedCalls": 40589, "reasoningOutputTokens": 23075825, "reasoningOutputTokensReportedCalls": 129846, "sourceReportedCostSum": 1344.5910701999999, "callsWithReportedCost": 129846 }, "diagnostics": { "nativeToolActions": 136132, "recognizedBashActions": 136132, "multiToolResponses": 5352 }, "candidates": [ { "program": "edit_compile_v3", "title": "Check syntax after a source edit", "proposals": 1574, "exactMatches": 30, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 26, "matchedUsage": { "calls": 30, "callsWithUsage": 30, "inputTokens": 947946, "inputTokensReportedCalls": 30, "outputTokens": 1403, "outputTokensReportedCalls": 30, "totalTokens": 949349, "totalTokensReportedCalls": 30, "cachedInputTokens": 815104, "cachedInputTokensReportedCalls": 30, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 372, "reasoningOutputTokensReportedCalls": 30, "sourceReportedCostSum": 0.3947587, "callsWithReportedCost": 30 }, "matchesWithReasoningOrProse": 9, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 530, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 232, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 557, "matchedUsage": { "calls": 30, "callsWithUsage": 30, "inputTokens": 947946, "inputTokensReportedCalls": 30, "outputTokens": 1403, "outputTokensReportedCalls": 30, "totalTokens": 949349, "totalTokensReportedCalls": 30, "cachedInputTokens": 815104, "cachedInputTokensReportedCalls": 30, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 372, "reasoningOutputTokensReportedCalls": 30, "sourceReportedCostSum": 0.3947587, "callsWithReportedCost": 30 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 188, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 67, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "inspect_unique_small_file_v3", "title": "Read a uniquely identified small file", "proposals": 4191, "exactMatches": 31, "coverageOnlyMatches": 0, "overlapExcluded": 0, "equivalentEOFMatches": 36, "matchedIssues": 30, "matchedUsage": { "calls": 31, "callsWithUsage": 31, "inputTokens": 330311, "inputTokensReportedCalls": 31, "outputTokens": 2952, "outputTokensReportedCalls": 31, "totalTokens": 333263, "totalTokensReportedCalls": 31, "cachedInputTokens": 213492, "cachedInputTokensReportedCalls": 26, "cacheCreationInputTokens": 878, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 1528, "reasoningOutputTokensReportedCalls": 31, "sourceReportedCostSum": 0.09275051999999999, "callsWithReportedCost": 31 }, "matchesWithReasoningOrProse": 31, "equivalentEOF": { "matchedUsage": { "calls": 36, "callsWithUsage": 36, "inputTokens": 516234, "inputTokensReportedCalls": 36, "outputTokens": 2792, "outputTokensReportedCalls": 36, "totalTokens": 519026, "totalTokensReportedCalls": 36, "cachedInputTokens": 362900, "cachedInputTokensReportedCalls": 35, "cacheCreationInputTokens": 153, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 1241, "reasoningOutputTokensReportedCalls": 36, "sourceReportedCostSum": 0.19355547, "callsWithReportedCost": 36 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 1820, "matchedUsage": { "calls": 15, "callsWithUsage": 15, "inputTokens": 111677, "inputTokensReportedCalls": 15, "outputTokens": 1081, "outputTokensReportedCalls": 15, "totalTokens": 112758, "totalTokensReportedCalls": 15, "cachedInputTokens": 48243, "cachedInputTokensReportedCalls": 15, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 265, "reasoningOutputTokensReportedCalls": 15, "sourceReportedCostSum": 0.02414007, "callsWithReportedCost": 15 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 1192, "matchedUsage": { "calls": 11, "callsWithUsage": 11, "inputTokens": 173788, "inputTokensReportedCalls": 11, "outputTokens": 650, "outputTokensReportedCalls": 11, "totalTokens": 174438, "totalTokensReportedCalls": 11, "cachedInputTokens": 122435, "cachedInputTokensReportedCalls": 6, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 366, "reasoningOutputTokensReportedCalls": 11, "sourceReportedCostSum": 0.03374825, "callsWithReportedCost": 11 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 660, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 7274, "inputTokensReportedCalls": 1, "outputTokens": 871, "outputTokensReportedCalls": 1, "totalTokens": 8145, "totalTokensReportedCalls": 1, "cachedInputTokens": 6144, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 838, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.0152467, "callsWithReportedCost": 1 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 82, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 437, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 37572, "inputTokensReportedCalls": 4, "outputTokens": 350, "outputTokensReportedCalls": 4, "totalTokens": 37922, "totalTokensReportedCalls": 4, "cachedInputTokens": 36670, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 878, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 59, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.0196155, "callsWithReportedCost": 4 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "missing_rg_literal_search_repair_v3", "title": "Repair a missing ripgrep command", "proposals": 11, "exactMatches": 9, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 9, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 35540, "inputTokensReportedCalls": 9, "outputTokens": 506, "outputTokensReportedCalls": 9, "totalTokens": 36046, "totalTokensReportedCalls": 9, "cachedInputTokens": 19456, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 116, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.0386358, "callsWithReportedCost": 9 }, "matchesWithReasoningOrProse": 9, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 11, "matchedUsage": { "calls": 9, "callsWithUsage": 9, "inputTokens": 35540, "inputTokensReportedCalls": 9, "outputTokens": 506, "outputTokensReportedCalls": 9, "totalTokens": 36046, "totalTokensReportedCalls": 9, "cachedInputTokens": 19456, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 116, "reasoningOutputTokensReportedCalls": 9, "sourceReportedCostSum": 0.0386358, "callsWithReportedCost": 9 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "registered_failed_test_rerun_v2", "title": "Rerun a failed assertion regression", "proposals": 3, "exactMatches": 0, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchesWithReasoningOrProse": 0, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 1, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 2, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "registered_test_rerun_v2", "title": "Rerun a previously passing test", "proposals": 92, "exactMatches": 18, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 17, "matchedUsage": { "calls": 18, "callsWithUsage": 18, "inputTokens": 444361, "inputTokensReportedCalls": 18, "outputTokens": 679, "outputTokensReportedCalls": 18, "totalTokens": 445040, "totalTokensReportedCalls": 18, "cachedInputTokens": 262107, "cachedInputTokensReportedCalls": 11, "cacheCreationInputTokens": 138, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 254, "reasoningOutputTokensReportedCalls": 18, "sourceReportedCostSum": 0.11198141, "callsWithReportedCost": 18 }, "matchesWithReasoningOrProse": 18, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 3, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 20755, "inputTokensReportedCalls": 1, "outputTokens": 52, "outputTokensReportedCalls": 1, "totalTokens": 20807, "totalTokensReportedCalls": 1, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 1, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.00541606, "callsWithReportedCost": 1 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 56, "matchedUsage": { "calls": 16, "callsWithUsage": 16, "inputTokens": 386456, "inputTokensReportedCalls": 16, "outputTokens": 533, "outputTokensReportedCalls": 16, "totalTokens": 386989, "totalTokensReportedCalls": 16, "cachedInputTokens": 225095, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 232, "reasoningOutputTokensReportedCalls": 16, "sourceReportedCostSum": 0.09353425, "callsWithReportedCost": 16 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 11, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 7, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 15, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 37150, "inputTokensReportedCalls": 1, "outputTokens": 94, "outputTokensReportedCalls": 1, "totalTokens": 37244, "totalTokensReportedCalls": 1, "cachedInputTokens": 37012, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 138, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 21, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.0130311, "callsWithReportedCost": 1 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "search_read_v2", "title": "Read context around the only search hit", "proposals": 3572, "exactMatches": 3, "coverageOnlyMatches": 1695, "overlapExcluded": 0, "matchedIssues": 3, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 13897, "inputTokensReportedCalls": 3, "outputTokens": 231, "outputTokensReportedCalls": 3, "totalTokens": 14128, "totalTokensReportedCalls": 3, "cachedInputTokens": 957, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 66, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.00359573, "callsWithReportedCost": 3 }, "matchesWithReasoningOrProse": 3, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 433, "matchedUsage": { "calls": 1695, "callsWithUsage": 1695, "inputTokens": 35831125, "inputTokensReportedCalls": 1695, "outputTokens": 140649, "outputTokensReportedCalls": 1695, "totalTokens": 35971774, "totalTokensReportedCalls": 1695, "cachedInputTokens": 24572806, "cachedInputTokensReportedCalls": 1420, "cacheCreationInputTokens": 42955, "cacheCreationInputTokensReportedCalls": 210, "reasoningOutputTokens": 52203, "reasoningOutputTokensReportedCalls": 1695, "sourceReportedCostSum": 9.388048660000003, "callsWithReportedCost": 1695 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 1681, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 13897, "inputTokensReportedCalls": 3, "outputTokens": 231, "outputTokensReportedCalls": 3, "totalTokens": 14128, "totalTokensReportedCalls": 3, "cachedInputTokens": 957, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 66, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.00359573, "callsWithReportedCost": 3 }, "coverageOnlyUsage": { "calls": 681, "callsWithUsage": 681, "inputTokens": 15160227, "inputTokensReportedCalls": 681, "outputTokens": 61108, "outputTokensReportedCalls": 681, "totalTokens": 15221335, "totalTokensReportedCalls": 681, "cachedInputTokens": 10114910, "cachedInputTokensReportedCalls": 681, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 18905, "reasoningOutputTokensReportedCalls": 681, "sourceReportedCostSum": 2.8577454099999993, "callsWithReportedCost": 681 } }, "gemini3flash": { "proposals": 1165, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 619, "callsWithUsage": 619, "inputTokens": 14130872, "inputTokensReportedCalls": 619, "outputTokens": 51746, "outputTokensReportedCalls": 619, "totalTokens": 14182618, "totalTokensReportedCalls": 619, "cachedInputTokens": 8443785, "cachedInputTokensReportedCalls": 344, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 29879, "reasoningOutputTokensReportedCalls": 619, "sourceReportedCostSum": 3.4209707499999964, "callsWithReportedCost": 619 } }, "gpt52": { "proposals": 311, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 185, "callsWithUsage": 185, "inputTokens": 3068199, "inputTokensReportedCalls": 185, "outputTokens": 7522, "outputTokensReportedCalls": 185, "totalTokens": 3075721, "totalTokensReportedCalls": 185, "cachedInputTokens": 2592512, "cachedInputTokensReportedCalls": 185, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 390, "reasoningOutputTokensReportedCalls": 185, "sourceReportedCostSum": 1.3914498499999997, "callsWithReportedCost": 185 } }, "opus45": { "proposals": 75, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 38, "callsWithUsage": 38, "inputTokens": 680297, "inputTokensReportedCalls": 38, "outputTokens": 3767, "outputTokensReportedCalls": 38, "totalTokens": 684064, "totalTokensReportedCalls": 38, "cachedInputTokens": 665501, "cachedInputTokensReportedCalls": 38, "cacheCreationInputTokens": 8091, "cacheCreationInputTokensReportedCalls": 38, "reasoningOutputTokens": 590, "reasoningOutputTokensReportedCalls": 38, "sourceReportedCostSum": 0.51101925, "callsWithReportedCost": 38 } }, "sonnet45": { "proposals": 340, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 172, "callsWithUsage": 172, "inputTokens": 2791530, "inputTokensReportedCalls": 172, "outputTokens": 16506, "outputTokensReportedCalls": 172, "totalTokens": 2808036, "totalTokensReportedCalls": 172, "cachedInputTokens": 2756098, "cachedInputTokensReportedCalls": 172, "cacheCreationInputTokens": 34864, "cacheCreationInputTokensReportedCalls": 172, "reasoningOutputTokens": 2439, "reasoningOutputTokensReportedCalls": 172, "sourceReportedCostSum": 1.2068633999999996, "callsWithReportedCost": 172 } } } }, { "program": "syntax_check_status_v3", "title": "Inspect status after a syntax check", "proposals": 382, "exactMatches": 42, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 42, "matchedUsage": { "calls": 42, "callsWithUsage": 42, "inputTokens": 1101960, "inputTokensReportedCalls": 42, "outputTokens": 11742, "outputTokensReportedCalls": 42, "totalTokens": 1113702, "totalTokensReportedCalls": 42, "cachedInputTokens": 1006720, "cachedInputTokensReportedCalls": 42, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 10445, "reasoningOutputTokensReportedCalls": 42, "sourceReportedCostSum": 0.5072339999999999, "callsWithReportedCost": 42 }, "matchesWithReasoningOrProse": 42, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 14, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 367, "matchedUsage": { "calls": 42, "callsWithUsage": 42, "inputTokens": 1101960, "inputTokensReportedCalls": 42, "outputTokens": 11742, "outputTokensReportedCalls": 42, "totalTokens": 1113702, "totalTokensReportedCalls": 42, "cachedInputTokens": 1006720, "cachedInputTokensReportedCalls": 42, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 10445, "reasoningOutputTokensReportedCalls": 42, "sourceReportedCostSum": 0.5072339999999999, "callsWithReportedCost": 42 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 1, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "write_then_run_v2", "title": "Write a script and run it", "proposals": 2238, "exactMatches": 438, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 166, "matchedUsage": { "calls": 438, "callsWithUsage": 438, "inputTokens": 6984083, "inputTokensReportedCalls": 438, "outputTokens": 33221, "outputTokensReportedCalls": 438, "totalTokens": 7017304, "totalTokensReportedCalls": 438, "cachedInputTokens": 5812335, "cachedInputTokensReportedCalls": 438, "cacheCreationInputTokens": 310874, "cacheCreationInputTokensReportedCalls": 343, "reasoningOutputTokens": 4062, "reasoningOutputTokensReportedCalls": 438, "sourceReportedCostSum": 3.4023556500000036, "callsWithReportedCost": 438 }, "matchesWithReasoningOrProse": 432, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 1651, "matchedUsage": { "calls": 95, "callsWithUsage": 95, "inputTokens": 1994924, "inputTokensReportedCalls": 95, "outputTokens": 5514, "outputTokensReportedCalls": 95, "totalTokens": 2000438, "totalTokensReportedCalls": 95, "cachedInputTokens": 1135363, "cachedInputTokensReportedCalls": 95, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 520, "reasoningOutputTokensReportedCalls": 95, "sourceReportedCostSum": 0.3963545499999999, "callsWithReportedCost": 95 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 2, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 13, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 54, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 42861, "inputTokensReportedCalls": 4, "outputTokens": 313, "outputTokensReportedCalls": 4, "totalTokens": 43174, "totalTokensReportedCalls": 4, "cachedInputTokens": 40295, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 2534, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 41, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.04397, "callsWithReportedCost": 4 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 518, "matchedUsage": { "calls": 339, "callsWithUsage": 339, "inputTokens": 4946298, "inputTokensReportedCalls": 339, "outputTokens": 27394, "outputTokensReportedCalls": 339, "totalTokens": 4973692, "totalTokensReportedCalls": 339, "cachedInputTokens": 4636677, "cachedInputTokensReportedCalls": 339, "cacheCreationInputTokens": 308340, "cacheCreationInputTokensReportedCalls": 339, "reasoningOutputTokens": 3501, "reasoningOutputTokensReportedCalls": 339, "sourceReportedCostSum": 2.962031100000003, "callsWithReportedCost": 339 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } } ], "union": { "matchedUsage": { "calls": 571, "callsWithUsage": 571, "inputTokens": 9858098, "inputTokensReportedCalls": 571, "outputTokens": 50734, "outputTokensReportedCalls": 571, "totalTokens": 9908832, "totalTokensReportedCalls": 571, "cachedInputTokens": 8130171, "cachedInputTokensReportedCalls": 559, "cacheCreationInputTokens": 311890, "cacheCreationInputTokensReportedCalls": 348, "reasoningOutputTokens": 16843, "reasoningOutputTokensReportedCalls": 571, "sourceReportedCostSum": 4.55131181, "callsWithReportedCost": 571 }, "matchedIssues": 240, "duplicateTargetsRemoved": 0, "dependentChainsExcluded": 0 }, "cohorts": { "deepseek32": { "trajectories": 500, "usage": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.92324824999977, "callsWithReportedCost": 43683 } }, "gemini3flash": { "trajectories": 498, "usage": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206540000024, "callsWithReportedCost": 28054 } }, "gpt52": { "trajectories": 500, "usage": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350884999995, "callsWithReportedCost": 17520 } }, "opus45": { "trajectories": 500, "usage": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.9539984999997, "callsWithReportedCost": 16446 } }, "sonnet45": { "trajectories": 500, "usage": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.94824920000025, "callsWithReportedCost": 24143 } } }, "limitations": [ "Policies developed on overlapping discovery subsets; see per-program split reports. This is not fully held-out validation.", "Match means predicted next whole singleton response, not proof that reasoning or task quality can be preserved.", "Recorded usage belongs to original calls; avoided usage is a conditional gross estimate. No routing/program/fallback cost or future cache effect is subtracted.", "Cached input and reasoning are subsets; never add them again to total tokens.", "Coverage-only search reads can add context and hit output caps; excluded from exact totals.", "Execution requires opt-in and state/hash checks not reconstructable from historical traces. No deployment savings credited.", "Only recognized bash-tool shapes are supported. Unsupported patterns abstain; parser errors are reported, not counted as zero-opportunity traces.", "Source-reported costs are metadata, not our spending or independently verified invoices.", "No source commands or scripts are executed, and no paid model requests are made." ], "referenceComparison": { "referenceManifestSha256": "cfeedd50bd348710bcb73f7e63c6488d8d7a662fa1c14d3da1d9ff4c317b9144", "knownIssueCount": 50, "previouslySampledIssues": { "reportedApiCalls": 13699, "trajectoriesWithReportedApiCalls": 250, "unrecordedResponseCount": 51, "trajectoriesWithCounterMismatch": 30, "trajectories": 250, "uniqueIssues": 50, "denominator": { "calls": 13648, "callsWithUsage": 13648, "inputTokens": 352941691, "inputTokensReportedCalls": 13648, "outputTokens": 4372569, "outputTokensReportedCalls": 13648, "totalTokens": 357314260, "totalTokensReportedCalls": 13648, "cachedInputTokens": 280834968, "cachedInputTokensReportedCalls": 12870, "cacheCreationInputTokens": 3140826, "cacheCreationInputTokensReportedCalls": 4005, "reasoningOutputTokens": 2390711, "reasoningOutputTokensReportedCalls": 13648, "sourceReportedCostSum": 137.49757549, "callsWithReportedCost": 13648 }, "diagnostics": { "nativeToolActions": 14434, "recognizedBashActions": 14434, "multiToolResponses": 665 }, "candidates": [ { "program": "edit_compile_v3", "title": "Check syntax after a source edit", "proposals": 196, "exactMatches": 3, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 3, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 102185, "inputTokensReportedCalls": 3, "outputTokens": 276, "outputTokensReportedCalls": 3, "totalTokens": 102461, "totalTokensReportedCalls": 3, "cachedInputTokens": 94848, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 173, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.033302149999999996, "callsWithReportedCost": 3 }, "matchesWithReasoningOrProse": 2, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 96, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 32, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 52, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 102185, "inputTokensReportedCalls": 3, "outputTokens": 276, "outputTokensReportedCalls": 3, "totalTokens": 102461, "totalTokensReportedCalls": 3, "cachedInputTokens": 94848, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 173, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.033302149999999996, "callsWithReportedCost": 3 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 12, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 4, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "inspect_unique_small_file_v3", "title": "Read a uniquely identified small file", "proposals": 444, "exactMatches": 4, "coverageOnlyMatches": 0, "overlapExcluded": 0, "equivalentEOFMatches": 4, "matchedIssues": 4, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 31885, "inputTokensReportedCalls": 4, "outputTokens": 1102, "outputTokensReportedCalls": 4, "totalTokens": 32987, "totalTokensReportedCalls": 4, "cachedInputTokens": 7101, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 899, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.02162807, "callsWithReportedCost": 4 }, "matchesWithReasoningOrProse": 4, "equivalentEOF": { "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 96100, "inputTokensReportedCalls": 4, "outputTokens": 207, "outputTokensReportedCalls": 4, "totalTokens": 96307, "totalTokensReportedCalls": 4, "cachedInputTokens": 70847, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 16, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.02037852, "callsWithReportedCost": 4 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 215, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 24611, "inputTokensReportedCalls": 3, "outputTokens": 231, "outputTokensReportedCalls": 3, "totalTokens": 24842, "totalTokensReportedCalls": 3, "cachedInputTokens": 957, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 61, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.00638137, "callsWithReportedCost": 3 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 120, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 65, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 7274, "inputTokensReportedCalls": 1, "outputTokens": 871, "outputTokensReportedCalls": 1, "totalTokens": 8145, "totalTokensReportedCalls": 1, "cachedInputTokens": 6144, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 838, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.0152467, "callsWithReportedCost": 1 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 5, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 39, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "missing_rg_literal_search_repair_v3", "title": "Repair a missing ripgrep command", "proposals": 1, "exactMatches": 1, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 1, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 8200, "inputTokensReportedCalls": 1, "outputTokens": 57, "outputTokensReportedCalls": 1, "totalTokens": 8257, "totalTokensReportedCalls": 1, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 13, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.015148, "callsWithReportedCost": 1 }, "matchesWithReasoningOrProse": 1, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 1, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 8200, "inputTokensReportedCalls": 1, "outputTokens": 57, "outputTokensReportedCalls": 1, "totalTokens": 8257, "totalTokensReportedCalls": 1, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 13, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.015148, "callsWithReportedCost": 1 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "registered_failed_test_rerun_v2", "title": "Rerun a failed assertion regression", "proposals": 0, "exactMatches": 0, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchesWithReasoningOrProse": 0, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "registered_test_rerun_v2", "title": "Rerun a previously passing test", "proposals": 5, "exactMatches": 2, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 2, "matchedUsage": { "calls": 2, "callsWithUsage": 2, "inputTokens": 28548, "inputTokensReportedCalls": 2, "outputTokens": 74, "outputTokensReportedCalls": 2, "totalTokens": 28622, "totalTokensReportedCalls": 2, "cachedInputTokens": 6410, "cachedInputTokensReportedCalls": 2, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 3, "reasoningOutputTokensReportedCalls": 2, "sourceReportedCostSum": 0.00649406, "callsWithReportedCost": 2 }, "matchesWithReasoningOrProse": 2, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 1, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 20755, "inputTokensReportedCalls": 1, "outputTokens": 52, "outputTokensReportedCalls": 1, "totalTokens": 20807, "totalTokensReportedCalls": 1, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 1, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.00541606, "callsWithReportedCost": 1 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 4, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 7793, "inputTokensReportedCalls": 1, "outputTokens": 22, "outputTokensReportedCalls": 1, "totalTokens": 7815, "totalTokensReportedCalls": 1, "cachedInputTokens": 6410, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 2, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.001078, "callsWithReportedCost": 1 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "search_read_v2", "title": "Read context around the only search hit", "proposals": 378, "exactMatches": 0, "coverageOnlyMatches": 149, "overlapExcluded": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchesWithReasoningOrProse": 0, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 41, "matchedUsage": { "calls": 149, "callsWithUsage": 149, "inputTokens": 3894651, "inputTokensReportedCalls": 149, "outputTokens": 12710, "outputTokensReportedCalls": 149, "totalTokens": 3907361, "totalTokensReportedCalls": 149, "cachedInputTokens": 2709066, "cachedInputTokensReportedCalls": 123, "cacheCreationInputTokens": 3208, "cacheCreationInputTokensReportedCalls": 14, "reasoningOutputTokens": 5103, "reasoningOutputTokensReportedCalls": 149, "sourceReportedCostSum": 0.92590296, "callsWithReportedCost": 149 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 200, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 64, "callsWithUsage": 64, "inputTokens": 1886107, "inputTokensReportedCalls": 64, "outputTokens": 5632, "outputTokensReportedCalls": 64, "totalTokens": 1891739, "totalTokensReportedCalls": 64, "cachedInputTokens": 1322632, "cachedInputTokensReportedCalls": 64, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 1683, "reasoningOutputTokensReportedCalls": 64, "sourceReportedCostSum": 0.34703846, "callsWithReportedCost": 64 } }, "gemini3flash": { "proposals": 118, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 55, "callsWithUsage": 55, "inputTokens": 1451625, "inputTokensReportedCalls": 55, "outputTokens": 5108, "outputTokensReportedCalls": 55, "totalTokens": 1456733, "totalTokensReportedCalls": 55, "cachedInputTokens": 866592, "cachedInputTokensReportedCalls": 29, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 3202, "reasoningOutputTokensReportedCalls": 55, "sourceReportedCostSum": 0.35117009999999993, "callsWithReportedCost": 55 } }, "gpt52": { "proposals": 28, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 16, "callsWithUsage": 16, "inputTokens": 302622, "inputTokensReportedCalls": 16, "outputTokens": 610, "outputTokensReportedCalls": 16, "totalTokens": 303232, "totalTokensReportedCalls": 16, "cachedInputTokens": 268800, "cachedInputTokensReportedCalls": 16, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 16, "sourceReportedCostSum": 0.1147685, "callsWithReportedCost": 16 } }, "opus45": { "proposals": 3, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 14109, "inputTokensReportedCalls": 1, "outputTokens": 104, "outputTokensReportedCalls": 1, "totalTokens": 14213, "totalTokensReportedCalls": 1, "cachedInputTokens": 13594, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 507, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 25, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.01260575, "callsWithReportedCost": 1 } }, "sonnet45": { "proposals": 29, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 13, "callsWithUsage": 13, "inputTokens": 240188, "inputTokensReportedCalls": 13, "outputTokens": 1256, "outputTokensReportedCalls": 13, "totalTokens": 241444, "totalTokensReportedCalls": 13, "cachedInputTokens": 237448, "cachedInputTokensReportedCalls": 13, "cacheCreationInputTokens": 2701, "cacheCreationInputTokensReportedCalls": 13, "reasoningOutputTokens": 193, "reasoningOutputTokensReportedCalls": 13, "sourceReportedCostSum": 0.10032015, "callsWithReportedCost": 13 } } } }, { "program": "syntax_check_status_v3", "title": "Inspect status after a syntax check", "proposals": 35, "exactMatches": 8, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 8, "matchedUsage": { "calls": 8, "callsWithUsage": 8, "inputTokens": 259567, "inputTokensReportedCalls": 8, "outputTokens": 2670, "outputTokensReportedCalls": 8, "totalTokens": 262237, "totalTokensReportedCalls": 8, "cachedInputTokens": 243968, "cachedInputTokensReportedCalls": 8, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 2438, "reasoningOutputTokensReportedCalls": 8, "sourceReportedCostSum": 0.10737264999999999, "callsWithReportedCost": 8 }, "matchesWithReasoningOrProse": 8, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 1, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 34, "matchedUsage": { "calls": 8, "callsWithUsage": 8, "inputTokens": 259567, "inputTokensReportedCalls": 8, "outputTokens": 2670, "outputTokensReportedCalls": 8, "totalTokens": 262237, "totalTokensReportedCalls": 8, "cachedInputTokens": 243968, "cachedInputTokensReportedCalls": 8, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 2438, "reasoningOutputTokensReportedCalls": 8, "sourceReportedCostSum": 0.10737264999999999, "callsWithReportedCost": 8 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "write_then_run_v2", "title": "Write a script and run it", "proposals": 208, "exactMatches": 48, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 17, "matchedUsage": { "calls": 48, "callsWithUsage": 48, "inputTokens": 775626, "inputTokensReportedCalls": 48, "outputTokens": 3579, "outputTokensReportedCalls": 48, "totalTokens": 779205, "totalTokensReportedCalls": 48, "cachedInputTokens": 668708, "cachedInputTokensReportedCalls": 48, "cacheCreationInputTokens": 23336, "cacheCreationInputTokensReportedCalls": 36, "reasoningOutputTokens": 404, "reasoningOutputTokensReportedCalls": 48, "sourceReportedCostSum": 0.32323180999999995, "callsWithReportedCost": 48 }, "matchesWithReasoningOrProse": 47, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 136, "matchedUsage": { "calls": 12, "callsWithUsage": 12, "inputTokens": 284335, "inputTokensReportedCalls": 12, "outputTokens": 721, "outputTokensReportedCalls": 12, "totalTokens": 285056, "totalTokensReportedCalls": 12, "cachedInputTokens": 200887, "cachedInputTokensReportedCalls": 12, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 81, "reasoningOutputTokensReportedCalls": 12, "sourceReportedCostSum": 0.052103510000000006, "callsWithReportedCost": 12 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 1, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 4, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 67, "matchedUsage": { "calls": 36, "callsWithUsage": 36, "inputTokens": 491291, "inputTokensReportedCalls": 36, "outputTokens": 2858, "outputTokensReportedCalls": 36, "totalTokens": 494149, "totalTokensReportedCalls": 36, "cachedInputTokens": 467821, "cachedInputTokensReportedCalls": 36, "cacheCreationInputTokens": 23336, "cacheCreationInputTokensReportedCalls": 36, "reasoningOutputTokens": 323, "reasoningOutputTokensReportedCalls": 36, "sourceReportedCostSum": 0.2711283, "callsWithReportedCost": 36 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } } ], "union": { "matchedUsage": { "calls": 66, "callsWithUsage": 66, "inputTokens": 1206011, "inputTokensReportedCalls": 66, "outputTokens": 7758, "outputTokensReportedCalls": 66, "totalTokens": 1213769, "totalTokensReportedCalls": 66, "cachedInputTokens": 1021035, "cachedInputTokensReportedCalls": 66, "cacheCreationInputTokens": 23336, "cacheCreationInputTokensReportedCalls": 36, "reasoningOutputTokens": 3930, "reasoningOutputTokensReportedCalls": 66, "sourceReportedCostSum": 0.5071767400000002, "callsWithReportedCost": 66 }, "matchedIssues": 30, "duplicateTargetsRemoved": 0, "dependentChainsExcluded": 0 }, "cohorts": { "deepseek32": { "trajectories": 50, "usage": { "calls": 4644, "callsWithUsage": 4644, "inputTokens": 130463507, "inputTokensReportedCalls": 4644, "outputTokens": 1397182, "outputTokensReportedCalls": 4644, "totalTokens": 131860689, "totalTokensReportedCalls": 4644, "cachedInputTokens": 91256698, "cachedInputTokensReportedCalls": 4644, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 815938, "reasoningOutputTokensReportedCalls": 4644, "sourceReportedCostSum": 24.45323884, "callsWithReportedCost": 4644 } }, "gemini3flash": { "trajectories": 50, "usage": { "calls": 3054, "callsWithUsage": 3054, "inputTokens": 99868157, "inputTokensReportedCalls": 3054, "outputTokens": 1069802, "outputTokensReportedCalls": 3054, "totalTokens": 100937959, "totalTokensReportedCalls": 3054, "cachedInputTokens": 75936909, "cachedInputTokensReportedCalls": 2276, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 785010, "reasoningOutputTokensReportedCalls": 3054, "sourceReportedCostSum": 18.971875449999995, "callsWithReportedCost": 3054 } }, "gpt52": { "trajectories": 50, "usage": { "calls": 1945, "callsWithUsage": 1945, "inputTokens": 40065726, "inputTokensReportedCalls": 1945, "outputTokens": 740263, "outputTokensReportedCalls": 1945, "totalTokens": 40805989, "totalTokensReportedCalls": 1945, "cachedInputTokens": 34515072, "cachedInputTokensReportedCalls": 1945, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 535811, "reasoningOutputTokensReportedCalls": 1945, "sourceReportedCostSum": 26.117464100000007, "callsWithReportedCost": 1945 } }, "opus45": { "trajectories": 50, "usage": { "calls": 1583, "callsWithUsage": 1583, "inputTokens": 30417869, "inputTokensReportedCalls": 1583, "outputTokens": 447608, "outputTokensReportedCalls": 1583, "totalTokens": 30865477, "totalTokensReportedCalls": 1583, "cachedInputTokens": 28807777, "cachedInputTokensReportedCalls": 1583, "cacheCreationInputTokens": 1339611, "cacheCreationInputTokensReportedCalls": 1583, "reasoningOutputTokens": 106806, "reasoningOutputTokensReportedCalls": 1583, "sourceReportedCostSum": 35.31906224999999, "callsWithReportedCost": 1583 } }, "sonnet45": { "trajectories": 50, "usage": { "calls": 2422, "callsWithUsage": 2422, "inputTokens": 52126432, "inputTokensReportedCalls": 2422, "outputTokens": 717714, "outputTokensReportedCalls": 2422, "totalTokens": 52844146, "totalTokensReportedCalls": 2422, "cachedInputTokens": 50318512, "cachedInputTokensReportedCalls": 2422, "cacheCreationInputTokens": 1801215, "cacheCreationInputTokensReportedCalls": 2422, "reasoningOutputTokens": 147146, "reasoningOutputTokensReportedCalls": 2422, "sourceReportedCostSum": 32.63593485, "callsWithReportedCost": 2422 } } } }, "otherIssues": { "reportedApiCalls": 116736, "trajectoriesWithReportedApiCalls": 2248, "unrecordedResponseCount": 538, "trajectoriesWithCounterMismatch": 266, "trajectories": 2248, "uniqueIssues": 450, "denominator": { "calls": 116198, "callsWithUsage": 116198, "inputTokens": 2848283997, "inputTokensReportedCalls": 116198, "outputTokens": 38005039, "outputTokensReportedCalls": 116198, "totalTokens": 2886289036, "totalTokensReportedCalls": 116198, "cachedInputTokens": 2218030906, "cachedInputTokensReportedCalls": 108358, "cacheCreationInputTokens": 28407074, "cacheCreationInputTokensReportedCalls": 36584, "reasoningOutputTokens": 20685114, "reasoningOutputTokensReportedCalls": 116198, "sourceReportedCostSum": 1207.0934947099988, "callsWithReportedCost": 116198 }, "diagnostics": { "nativeToolActions": 121698, "recognizedBashActions": 121698, "multiToolResponses": 4687 }, "candidates": [ { "program": "edit_compile_v3", "title": "Check syntax after a source edit", "proposals": 1378, "exactMatches": 27, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 23, "matchedUsage": { "calls": 27, "callsWithUsage": 27, "inputTokens": 845761, "inputTokensReportedCalls": 27, "outputTokens": 1127, "outputTokensReportedCalls": 27, "totalTokens": 846888, "totalTokensReportedCalls": 27, "cachedInputTokens": 720256, "cachedInputTokensReportedCalls": 27, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 199, "reasoningOutputTokensReportedCalls": 27, "sourceReportedCostSum": 0.36145655000000004, "callsWithReportedCost": 27 }, "matchesWithReasoningOrProse": 7, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 434, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 200, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 505, "matchedUsage": { "calls": 27, "callsWithUsage": 27, "inputTokens": 845761, "inputTokensReportedCalls": 27, "outputTokens": 1127, "outputTokensReportedCalls": 27, "totalTokens": 846888, "totalTokensReportedCalls": 27, "cachedInputTokens": 720256, "cachedInputTokensReportedCalls": 27, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 199, "reasoningOutputTokensReportedCalls": 27, "sourceReportedCostSum": 0.36145655000000004, "callsWithReportedCost": 27 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 176, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 63, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "inspect_unique_small_file_v3", "title": "Read a uniquely identified small file", "proposals": 3747, "exactMatches": 27, "coverageOnlyMatches": 0, "overlapExcluded": 0, "equivalentEOFMatches": 32, "matchedIssues": 26, "matchedUsage": { "calls": 27, "callsWithUsage": 27, "inputTokens": 298426, "inputTokensReportedCalls": 27, "outputTokens": 1850, "outputTokensReportedCalls": 27, "totalTokens": 300276, "totalTokensReportedCalls": 27, "cachedInputTokens": 206391, "cachedInputTokensReportedCalls": 22, "cacheCreationInputTokens": 878, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 629, "reasoningOutputTokensReportedCalls": 27, "sourceReportedCostSum": 0.07112245, "callsWithReportedCost": 27 }, "matchesWithReasoningOrProse": 27, "equivalentEOF": { "matchedUsage": { "calls": 32, "callsWithUsage": 32, "inputTokens": 420134, "inputTokensReportedCalls": 32, "outputTokens": 2585, "outputTokensReportedCalls": 32, "totalTokens": 422719, "totalTokensReportedCalls": 32, "cachedInputTokens": 292053, "cachedInputTokensReportedCalls": 31, "cacheCreationInputTokens": 153, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 1225, "reasoningOutputTokensReportedCalls": 32, "sourceReportedCostSum": 0.17317695, "callsWithReportedCost": 32 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 1605, "matchedUsage": { "calls": 12, "callsWithUsage": 12, "inputTokens": 87066, "inputTokensReportedCalls": 12, "outputTokens": 850, "outputTokensReportedCalls": 12, "totalTokens": 87916, "totalTokensReportedCalls": 12, "cachedInputTokens": 47286, "cachedInputTokensReportedCalls": 12, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 204, "reasoningOutputTokensReportedCalls": 12, "sourceReportedCostSum": 0.0177587, "callsWithReportedCost": 12 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 1072, "matchedUsage": { "calls": 11, "callsWithUsage": 11, "inputTokens": 173788, "inputTokensReportedCalls": 11, "outputTokens": 650, "outputTokensReportedCalls": 11, "totalTokens": 174438, "totalTokensReportedCalls": 11, "cachedInputTokens": 122435, "cachedInputTokensReportedCalls": 6, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 366, "reasoningOutputTokensReportedCalls": 11, "sourceReportedCostSum": 0.03374825, "callsWithReportedCost": 11 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 595, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 77, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 398, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 37572, "inputTokensReportedCalls": 4, "outputTokens": 350, "outputTokensReportedCalls": 4, "totalTokens": 37922, "totalTokensReportedCalls": 4, "cachedInputTokens": 36670, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 878, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 59, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.0196155, "callsWithReportedCost": 4 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "missing_rg_literal_search_repair_v3", "title": "Repair a missing ripgrep command", "proposals": 10, "exactMatches": 8, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 8, "matchedUsage": { "calls": 8, "callsWithUsage": 8, "inputTokens": 27340, "inputTokensReportedCalls": 8, "outputTokens": 449, "outputTokensReportedCalls": 8, "totalTokens": 27789, "totalTokensReportedCalls": 8, "cachedInputTokens": 19456, "cachedInputTokensReportedCalls": 8, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 103, "reasoningOutputTokensReportedCalls": 8, "sourceReportedCostSum": 0.023487800000000003, "callsWithReportedCost": 8 }, "matchesWithReasoningOrProse": 8, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 10, "matchedUsage": { "calls": 8, "callsWithUsage": 8, "inputTokens": 27340, "inputTokensReportedCalls": 8, "outputTokens": 449, "outputTokensReportedCalls": 8, "totalTokens": 27789, "totalTokensReportedCalls": 8, "cachedInputTokens": 19456, "cachedInputTokensReportedCalls": 8, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 103, "reasoningOutputTokensReportedCalls": 8, "sourceReportedCostSum": 0.023487800000000003, "callsWithReportedCost": 8 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "registered_failed_test_rerun_v2", "title": "Rerun a failed assertion regression", "proposals": 3, "exactMatches": 0, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchesWithReasoningOrProse": 0, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 1, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 2, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "registered_test_rerun_v2", "title": "Rerun a previously passing test", "proposals": 87, "exactMatches": 16, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 15, "matchedUsage": { "calls": 16, "callsWithUsage": 16, "inputTokens": 415813, "inputTokensReportedCalls": 16, "outputTokens": 605, "outputTokensReportedCalls": 16, "totalTokens": 416418, "totalTokensReportedCalls": 16, "cachedInputTokens": 255697, "cachedInputTokensReportedCalls": 9, "cacheCreationInputTokens": 138, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 251, "reasoningOutputTokensReportedCalls": 16, "sourceReportedCostSum": 0.10548735000000001, "callsWithReportedCost": 16 }, "matchesWithReasoningOrProse": 16, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 2, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 52, "matchedUsage": { "calls": 15, "callsWithUsage": 15, "inputTokens": 378663, "inputTokensReportedCalls": 15, "outputTokens": 511, "outputTokensReportedCalls": 15, "totalTokens": 379174, "totalTokensReportedCalls": 15, "cachedInputTokens": 218685, "cachedInputTokensReportedCalls": 8, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 230, "reasoningOutputTokensReportedCalls": 15, "sourceReportedCostSum": 0.09245625, "callsWithReportedCost": 15 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 11, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 7, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 15, "matchedUsage": { "calls": 1, "callsWithUsage": 1, "inputTokens": 37150, "inputTokensReportedCalls": 1, "outputTokens": 94, "outputTokensReportedCalls": 1, "totalTokens": 37244, "totalTokensReportedCalls": 1, "cachedInputTokens": 37012, "cachedInputTokensReportedCalls": 1, "cacheCreationInputTokens": 138, "cacheCreationInputTokensReportedCalls": 1, "reasoningOutputTokens": 21, "reasoningOutputTokensReportedCalls": 1, "sourceReportedCostSum": 0.0130311, "callsWithReportedCost": 1 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "search_read_v2", "title": "Read context around the only search hit", "proposals": 3194, "exactMatches": 3, "coverageOnlyMatches": 1546, "overlapExcluded": 0, "matchedIssues": 3, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 13897, "inputTokensReportedCalls": 3, "outputTokens": 231, "outputTokensReportedCalls": 3, "totalTokens": 14128, "totalTokensReportedCalls": 3, "cachedInputTokens": 957, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 66, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.00359573, "callsWithReportedCost": 3 }, "matchesWithReasoningOrProse": 3, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 392, "matchedUsage": { "calls": 1546, "callsWithUsage": 1546, "inputTokens": 31936474, "inputTokensReportedCalls": 1546, "outputTokens": 127939, "outputTokensReportedCalls": 1546, "totalTokens": 32064413, "totalTokensReportedCalls": 1546, "cachedInputTokens": 21863740, "cachedInputTokensReportedCalls": 1297, "cacheCreationInputTokens": 39747, "cacheCreationInputTokensReportedCalls": 196, "reasoningOutputTokens": 47100, "reasoningOutputTokensReportedCalls": 1546, "sourceReportedCostSum": 8.462145700000002, "callsWithReportedCost": 1546 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 1481, "matchedUsage": { "calls": 3, "callsWithUsage": 3, "inputTokens": 13897, "inputTokensReportedCalls": 3, "outputTokens": 231, "outputTokensReportedCalls": 3, "totalTokens": 14128, "totalTokensReportedCalls": 3, "cachedInputTokens": 957, "cachedInputTokensReportedCalls": 3, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 66, "reasoningOutputTokensReportedCalls": 3, "sourceReportedCostSum": 0.00359573, "callsWithReportedCost": 3 }, "coverageOnlyUsage": { "calls": 617, "callsWithUsage": 617, "inputTokens": 13274120, "inputTokensReportedCalls": 617, "outputTokens": 55476, "outputTokensReportedCalls": 617, "totalTokens": 13329596, "totalTokensReportedCalls": 617, "cachedInputTokens": 8792278, "cachedInputTokensReportedCalls": 617, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 17222, "reasoningOutputTokensReportedCalls": 617, "sourceReportedCostSum": 2.5107069499999977, "callsWithReportedCost": 617 } }, "gemini3flash": { "proposals": 1047, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 564, "callsWithUsage": 564, "inputTokens": 12679247, "inputTokensReportedCalls": 564, "outputTokens": 46638, "outputTokensReportedCalls": 564, "totalTokens": 12725885, "totalTokensReportedCalls": 564, "cachedInputTokens": 7577193, "cachedInputTokensReportedCalls": 315, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 26677, "reasoningOutputTokensReportedCalls": 564, "sourceReportedCostSum": 3.069800649999997, "callsWithReportedCost": 564 } }, "gpt52": { "proposals": 283, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 169, "callsWithUsage": 169, "inputTokens": 2765577, "inputTokensReportedCalls": 169, "outputTokens": 6912, "outputTokensReportedCalls": 169, "totalTokens": 2772489, "totalTokensReportedCalls": 169, "cachedInputTokens": 2323712, "cachedInputTokensReportedCalls": 169, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 390, "reasoningOutputTokensReportedCalls": 169, "sourceReportedCostSum": 1.2766813499999998, "callsWithReportedCost": 169 } }, "opus45": { "proposals": 72, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 37, "callsWithUsage": 37, "inputTokens": 666188, "inputTokensReportedCalls": 37, "outputTokens": 3663, "outputTokensReportedCalls": 37, "totalTokens": 669851, "totalTokensReportedCalls": 37, "cachedInputTokens": 651907, "cachedInputTokensReportedCalls": 37, "cacheCreationInputTokens": 7584, "cacheCreationInputTokensReportedCalls": 37, "reasoningOutputTokens": 565, "reasoningOutputTokensReportedCalls": 37, "sourceReportedCostSum": 0.49841349999999995, "callsWithReportedCost": 37 } }, "sonnet45": { "proposals": 311, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 159, "callsWithUsage": 159, "inputTokens": 2551342, "inputTokensReportedCalls": 159, "outputTokens": 15250, "outputTokensReportedCalls": 159, "totalTokens": 2566592, "totalTokensReportedCalls": 159, "cachedInputTokens": 2518650, "cachedInputTokensReportedCalls": 159, "cacheCreationInputTokens": 32163, "cacheCreationInputTokensReportedCalls": 159, "reasoningOutputTokens": 2246, "reasoningOutputTokensReportedCalls": 159, "sourceReportedCostSum": 1.1065432499999999, "callsWithReportedCost": 159 } } } }, { "program": "syntax_check_status_v3", "title": "Inspect status after a syntax check", "proposals": 347, "exactMatches": 34, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 34, "matchedUsage": { "calls": 34, "callsWithUsage": 34, "inputTokens": 842393, "inputTokensReportedCalls": 34, "outputTokens": 9072, "outputTokensReportedCalls": 34, "totalTokens": 851465, "totalTokensReportedCalls": 34, "cachedInputTokens": 762752, "cachedInputTokensReportedCalls": 34, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 8007, "reasoningOutputTokensReportedCalls": 34, "sourceReportedCostSum": 0.3998613499999999, "callsWithReportedCost": 34 }, "matchesWithReasoningOrProse": 34, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 13, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 333, "matchedUsage": { "calls": 34, "callsWithUsage": 34, "inputTokens": 842393, "inputTokensReportedCalls": 34, "outputTokens": 9072, "outputTokensReportedCalls": 34, "totalTokens": 851465, "totalTokensReportedCalls": 34, "cachedInputTokens": 762752, "cachedInputTokensReportedCalls": 34, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 8007, "reasoningOutputTokensReportedCalls": 34, "sourceReportedCostSum": 0.3998613499999999, "callsWithReportedCost": 34 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 1, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } }, { "program": "write_then_run_v2", "title": "Write a script and run it", "proposals": 2030, "exactMatches": 390, "coverageOnlyMatches": 0, "overlapExcluded": 0, "matchedIssues": 149, "matchedUsage": { "calls": 390, "callsWithUsage": 390, "inputTokens": 6208457, "inputTokensReportedCalls": 390, "outputTokens": 29642, "outputTokensReportedCalls": 390, "totalTokens": 6238099, "totalTokensReportedCalls": 390, "cachedInputTokens": 5143627, "cachedInputTokensReportedCalls": 390, "cacheCreationInputTokens": 287538, "cacheCreationInputTokensReportedCalls": 307, "reasoningOutputTokens": 3658, "reasoningOutputTokensReportedCalls": 390, "sourceReportedCostSum": 3.079123840000003, "callsWithReportedCost": 390 }, "matchesWithReasoningOrProse": 385, "equivalentEOF": { "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true }, "coverageOnly": { "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "excludedFromExactUnion": true, "extraContextCostMeasured": false }, "cohorts": { "deepseek32": { "proposals": 1515, "matchedUsage": { "calls": 83, "callsWithUsage": 83, "inputTokens": 1710589, "inputTokensReportedCalls": 83, "outputTokens": 4793, "outputTokensReportedCalls": 83, "totalTokens": 1715382, "totalTokensReportedCalls": 83, "cachedInputTokens": 934476, "cachedInputTokensReportedCalls": 83, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 439, "reasoningOutputTokensReportedCalls": 83, "sourceReportedCostSum": 0.34425103999999995, "callsWithReportedCost": 83 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gemini3flash": { "proposals": 2, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "gpt52": { "proposals": 12, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "opus45": { "proposals": 50, "matchedUsage": { "calls": 4, "callsWithUsage": 4, "inputTokens": 42861, "inputTokensReportedCalls": 4, "outputTokens": 313, "outputTokensReportedCalls": 4, "totalTokens": 43174, "totalTokensReportedCalls": 4, "cachedInputTokens": 40295, "cachedInputTokensReportedCalls": 4, "cacheCreationInputTokens": 2534, "cacheCreationInputTokensReportedCalls": 4, "reasoningOutputTokens": 41, "reasoningOutputTokensReportedCalls": 4, "sourceReportedCostSum": 0.04397, "callsWithReportedCost": 4 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } }, "sonnet45": { "proposals": 451, "matchedUsage": { "calls": 303, "callsWithUsage": 303, "inputTokens": 4455007, "inputTokensReportedCalls": 303, "outputTokens": 24536, "outputTokensReportedCalls": 303, "totalTokens": 4479543, "totalTokensReportedCalls": 303, "cachedInputTokens": 4168856, "cachedInputTokensReportedCalls": 303, "cacheCreationInputTokens": 285004, "cacheCreationInputTokensReportedCalls": 303, "reasoningOutputTokens": 3178, "reasoningOutputTokensReportedCalls": 303, "sourceReportedCostSum": 2.6909028000000026, "callsWithReportedCost": 303 }, "coverageOnlyUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 } } } } ], "union": { "matchedUsage": { "calls": 505, "callsWithUsage": 505, "inputTokens": 8652087, "inputTokensReportedCalls": 505, "outputTokens": 42976, "outputTokensReportedCalls": 505, "totalTokens": 8695063, "totalTokensReportedCalls": 505, "cachedInputTokens": 7109136, "cachedInputTokensReportedCalls": 493, "cacheCreationInputTokens": 288554, "cacheCreationInputTokensReportedCalls": 312, "reasoningOutputTokens": 12913, "reasoningOutputTokensReportedCalls": 505, "sourceReportedCostSum": 4.04413507, "callsWithReportedCost": 505 }, "matchedIssues": 210, "duplicateTargetsRemoved": 0, "dependentChainsExcluded": 0 }, "cohorts": { "deepseek32": { "trajectories": 450, "usage": { "calls": 39039, "callsWithUsage": 39039, "inputTokens": 1047267043, "inputTokensReportedCalls": 39039, "outputTokens": 12044791, "outputTokensReportedCalls": 39039, "totalTokens": 1059311834, "totalTokensReportedCalls": 39039, "cachedInputTokens": 707719005, "cachedInputTokensReportedCalls": 39039, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7125176, "reasoningOutputTokensReportedCalls": 39039, "sourceReportedCostSum": 199.47000940999985, "callsWithReportedCost": 39039 } }, "gemini3flash": { "trajectories": 448, "usage": { "calls": 25000, "callsWithUsage": 25000, "inputTokens": 748058778, "inputTokensReportedCalls": 25000, "outputTokens": 9001555, "outputTokensReportedCalls": 25000, "totalTokens": 757060333, "totalTokensReportedCalls": 25000, "cachedInputTokens": 537830809, "cachedInputTokensReportedCalls": 17160, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 6764895, "reasoningOutputTokensReportedCalls": 25000, "sourceReportedCostSum": 159.01018995000007, "callsWithReportedCost": 25000 } }, "gpt52": { "trajectories": 450, "usage": { "calls": 15575, "callsWithUsage": 15575, "inputTokens": 282532317, "inputTokensReportedCalls": 15575, "outputTokens": 5952643, "outputTokensReportedCalls": 15575, "totalTokens": 288484960, "totalTokensReportedCalls": 15575, "cachedInputTokens": 233080960, "cachedInputTokensReportedCalls": 15575, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4335320, "reasoningOutputTokensReportedCalls": 15575, "sourceReportedCostSum": 210.66604475, "callsWithReportedCost": 15575 } }, "opus45": { "trajectories": 450, "usage": { "calls": 14863, "callsWithUsage": 14863, "inputTokens": 300204350, "inputTokensReportedCalls": 14863, "outputTokens": 4358492, "outputTokensReportedCalls": 14863, "totalTokens": 304562842, "totalTokensReportedCalls": 14863, "cachedInputTokens": 285287525, "cachedInputTokensReportedCalls": 14863, "cacheCreationInputTokens": 12355799, "cacheCreationInputTokensReportedCalls": 14863, "reasoningOutputTokens": 1086604, "reasoningOutputTokensReportedCalls": 14863, "sourceReportedCostSum": 341.63493624999967, "callsWithReportedCost": 14863 } }, "sonnet45": { "trajectories": 450, "usage": { "calls": 21721, "callsWithUsage": 21721, "inputTokens": 470221509, "inputTokensReportedCalls": 21721, "outputTokens": 6647558, "outputTokensReportedCalls": 21721, "totalTokens": 476869067, "totalTokensReportedCalls": 21721, "cachedInputTokens": 454112607, "cachedInputTokensReportedCalls": 21721, "cacheCreationInputTokens": 16051275, "cacheCreationInputTokensReportedCalls": 21721, "reasoningOutputTokens": 1373119, "reasoningOutputTokensReportedCalls": 21721, "sourceReportedCostSum": 296.3123143500001, "callsWithReportedCost": 21721 } } } }, "interpretation": "Issue-disjoint from the specified earlier sample, not a claim of untouched external validation." } }