{ "version": 3, "program": "syntax_check_status_v3", "title": "Inspect repository status after a syntax check", "kind": "conditional-gross-replay-estimate-not-measured-savings", "policy": { "root": "/testbed", "trigger": "Successful standalone python -m py_compile or python -m compileall -q on explicit .py files", "nextArgv": [ "git", "status", "--porcelain" ], "maxOutputBytes": 32768, "timeoutSeconds": 10, "requiresRegisteredRepository": true }, "trajectories": 2498, "issues": 500, "proposals": 382, "matchedCalls": 42, "unmatchedProposals": 340, "matchedIssues": 42, "matchPercent": 10.99476439790576, "matchedUsage": { "calls": 42, "callsWithUsage": 42, "inputTokens": 1101960, "inputTokensReportedCalls": 42, "outputTokens": 11742, "outputTokensReportedCalls": 42, "totalTokens": 1113702, "totalTokensReportedCalls": 42, "cachedInputTokens": 1006720, "cachedInputTokensReportedCalls": 42, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 10445, "reasoningOutputTokensReportedCalls": 42, "sourceReportedCostSum": 0.5072339999999999, "callsWithReportedCost": 42 }, "denominator": { "calls": 129846, "callsWithUsage": 129846, "inputTokens": 3201225688, "inputTokensReportedCalls": 129846, "outputTokens": 42377608, "outputTokensReportedCalls": 129846, "totalTokens": 3243603296, "totalTokensReportedCalls": 129846, "cachedInputTokens": 2498865874, "cachedInputTokensReportedCalls": 121228, "cacheCreationInputTokens": 31547900, "cacheCreationInputTokensReportedCalls": 40589, "reasoningOutputTokens": 23075825, "reasoningOutputTokensReportedCalls": 129846, "sourceReportedCostSum": 1344.591070199989, "callsWithReportedCost": 129846 }, "callPercent": 0.03234600988863731, "tokenPercent": 0.03433533321949122, "matchesWithReasoningOrProse": 42, "diagnostics": { "recognizedSyntaxChecks": 385, "nextCommandDiffers": 338, "nextNotSingletonBashWithResult": 2 }, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "proposals": 0, "matchedCalls": 0, "unmatchedProposals": 0, "matchedIssues": 0, "matchPercent": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "gemini3flash": { "trajectories": 498, "issues": 498, "proposals": 14, "matchedCalls": 0, "unmatchedProposals": 14, "matchedIssues": 0, "matchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSyntaxChecks": 14, "nextCommandDiffers": 14 } }, "gpt52": { "trajectories": 500, "issues": 500, "proposals": 367, "matchedCalls": 42, "unmatchedProposals": 325, "matchedIssues": 42, "matchPercent": 11.444141689373296, "matchedUsage": { "calls": 42, "callsWithUsage": 42, "inputTokens": 1101960, "inputTokensReportedCalls": 42, "outputTokens": 11742, "outputTokensReportedCalls": 42, "totalTokens": 1113702, "totalTokensReportedCalls": 42, "cachedInputTokens": 1006720, "cachedInputTokensReportedCalls": 42, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 10445, "reasoningOutputTokensReportedCalls": 42, "sourceReportedCostSum": 0.5072339999999999, "callsWithReportedCost": 42 }, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "callPercent": 0.23972602739726026, "tokenPercent": 0.33821215049551817, "matchesWithReasoningOrProse": 42, "diagnostics": { "recognizedSyntaxChecks": 370, "nextCommandDiffers": 323, "nextNotSingletonBashWithResult": 2 } }, "opus45": { "trajectories": 500, "issues": 500, "proposals": 0, "matchedCalls": 0, "unmatchedProposals": 0, "matchedIssues": 0, "matchPercent": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "sonnet45": { "trajectories": 500, "issues": 500, "proposals": 1, "matchedCalls": 0, "unmatchedProposals": 1, "matchedIssues": 0, "matchPercent": 0.0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "callPercent": 0.0, "tokenPercent": 0.0, "matchesWithReasoningOrProse": 0, "diagnostics": { "recognizedSyntaxChecks": 1, "nextCommandDiffers": 1 } } }, "byPartition": { "discovery": { "trajectories": 784, "issues": 157, "proposals": 118, "matchedCalls": 10, "unmatchedProposals": 108, "matchedIssues": 10, "matchPercent": 8.474576271186441, "matchedUsage": { "calls": 10, "callsWithUsage": 10, "inputTokens": 275040, "inputTokensReportedCalls": 10, "outputTokens": 2766, "outputTokensReportedCalls": 10, "totalTokens": 277806, "totalTokensReportedCalls": 10, "cachedInputTokens": 264576, "cachedInputTokensReportedCalls": 10, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 2397, "reasoningOutputTokensReportedCalls": 10, "sourceReportedCostSum": 0.10333679999999999, "callsWithReportedCost": 10 }, "denominator": { "calls": 39747, "callsWithUsage": 39747, "inputTokens": 963762231, "inputTokensReportedCalls": 39747, "outputTokens": 13087687, "outputTokensReportedCalls": 39747, "totalTokens": 976849918, "totalTokensReportedCalls": 39747, "cachedInputTokens": 748649310, "cachedInputTokensReportedCalls": 37079, "cacheCreationInputTokens": 9560939, "cacheCreationInputTokensReportedCalls": 12290, "reasoningOutputTokens": 7163382, "reasoningOutputTokensReportedCalls": 39747, "sourceReportedCostSum": 408.1896112699991, "callsWithReportedCost": 39747 }, "callPercent": 0.025159131506780386, "tokenPercent": 0.028438964356856302, "matchesWithReasoningOrProse": 10, "diagnostics": { "recognizedSyntaxChecks": 118, "nextCommandDiffers": 108 } }, "validation": { "trajectories": 1714, "issues": 343, "proposals": 264, "matchedCalls": 32, "unmatchedProposals": 232, "matchedIssues": 32, "matchPercent": 12.121212121212121, "matchedUsage": { "calls": 32, "callsWithUsage": 32, "inputTokens": 826920, "inputTokensReportedCalls": 32, "outputTokens": 8976, "outputTokensReportedCalls": 32, "totalTokens": 835896, "totalTokensReportedCalls": 32, "cachedInputTokens": 742144, "cachedInputTokensReportedCalls": 32, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 8048, "reasoningOutputTokensReportedCalls": 32, "sourceReportedCostSum": 0.40389719999999996, "callsWithReportedCost": 32 }, "denominator": { "calls": 90099, "callsWithUsage": 90099, "inputTokens": 2237463457, "inputTokensReportedCalls": 90099, "outputTokens": 29289921, "outputTokensReportedCalls": 90099, "totalTokens": 2266753378, "totalTokensReportedCalls": 90099, "cachedInputTokens": 1750216564, "cachedInputTokensReportedCalls": 84149, "cacheCreationInputTokens": 21986961, "cacheCreationInputTokensReportedCalls": 28299, "reasoningOutputTokens": 15912443, "reasoningOutputTokensReportedCalls": 90099, "sourceReportedCostSum": 936.4014589299964, "callsWithReportedCost": 90099 }, "callPercent": 0.03551648741939422, "tokenPercent": 0.036876354000960045, "matchesWithReasoningOrProse": 32, "diagnostics": { "recognizedSyntaxChecks": 267, "nextCommandDiffers": 230, "nextNotSingletonBashWithResult": 2 } } }, "partitionMethod": "Discovery is union of SHA256(autotoken-v3:instance_id)%5==0 and initially inspected SHA256(diff-check-v3:instance_id)%5==0; validation excludes both. Same issue grouped across models.", "selectionNote": "Only discovery next actions were inspected to choose syntax-check -> fixed porcelain status. Policy frozen before replaying validation. Discovery prefix changed during coordination; union prevents inspected issues being relabeled validation. Corpus was previously used in project research, so not untouched external validation.", "limitations": [ "Only whole singleton next responses with observed successful bounded output count. No partial shell commands or redirected/bundled status calls credited.", "Root repository must be registered and still present; replay does not reconstruct historical filesystem state.", "Git status arguments/cwd match, not a reconstructed execution/output equivalence proof.", "All potential saved calls may contain useful reasoning; task quality remains untested.", "Successful compilation is syntax verification, not proof the bug is fixed or tests pass.", "Proposal precision is descriptive; unmatched proposals show that automatic routing is not justified universally.", "Recorded input/output/cache usage belongs to original calls. Savings are conditional gross estimates before router/program/fallback costs and future cache effects.", "Cached input and reasoning output are subsets, never added again.", "No recorded command was executed and no paid inference was used." ], "sourceManifestSha256": "73cccd8079f5c3150015207a7d102f1a4409c7996f0ea9ae1305f1b45239247b", "plannerSha256": "fdce9109b7df03d6043f090608248e29100adcf7132b3261f9a232b462e08cb5" }