{ "version": 5, "program": "explicit_finish_after_verification_v5", "title": "Finish only under an explicit registered verification policy", "kind": "hypothetical-explicit-opt-in-completion-scenario", "policy": { "root": "/testbed", "maxCommandBytes": 4096, "maxOutputBytes": 32768, "maxVerificationHistoryTurns": 4, "requiredEvidence": [ "registered exact pytest argv succeeds with positive passed-count summary", "subsequent successful full-tree git diff --check", "no intervening unknown or mutating action" ], "requiresExplicitRegisteredCompletionPolicy": true, "requiresRegisteredRequiredTestCommand": true, "requiresUnchangedTreeAndEnvironment": true, "requiresNoUnverifiedStagedOrUntrackedChanges": true, "historicalOptInPresent": false, "deploymentCreditCalls": 0, "terminalCommand": "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT", "allowFinalProseInstead": false }, "trajectories": 2498, "issues": 500, "denominator": { "calls": 129846, "callsWithUsage": 129846, "inputTokens": 3201225688, "inputTokensReportedCalls": 129846, "outputTokens": 42377608, "outputTokensReportedCalls": 129846, "totalTokens": 3243603296, "totalTokensReportedCalls": 129846, "cachedInputTokens": 2498865874, "cachedInputTokensReportedCalls": 121228, "cacheCreationInputTokens": 31547900, "cacheCreationInputTokensReportedCalls": 40589, "reasoningOutputTokens": 23075825, "reasoningOutputTokensReportedCalls": 129846, "sourceReportedCostSum": 1344.591070199989, "callsWithReportedCost": 129846 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "diffCheckLikeTurns": 40, "supportedPassedFullTreeCheckTurns": 7 }, "byPartition": { "discovery": { "trajectories": 550, "issues": 110, "denominator": { "calls": 28862, "callsWithUsage": 28862, "inputTokens": 702102702, "inputTokensReportedCalls": 28862, "outputTokens": 9426963, "outputTokensReportedCalls": 28862, "totalTokens": 711529665, "totalTokensReportedCalls": 28862, "cachedInputTokens": 543584191, "cachedInputTokensReportedCalls": 27013, "cacheCreationInputTokens": 6709415, "cacheCreationInputTokensReportedCalls": 8757, "reasoningOutputTokens": 5220878, "reasoningOutputTokensReportedCalls": 28862, "sourceReportedCostSum": 291.47569229000186, "callsWithReportedCost": 28862 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "diffCheckLikeTurns": 13, "supportedPassedFullTreeCheckTurns": 2 } }, "validation": { "trajectories": 1948, "issues": 390, "denominator": { "calls": 100984, "callsWithUsage": 100984, "inputTokens": 2499122986, "inputTokensReportedCalls": 100984, "outputTokens": 32950645, "outputTokensReportedCalls": 100984, "totalTokens": 2532073631, "totalTokensReportedCalls": 100984, "cachedInputTokens": 1955281683, "cachedInputTokensReportedCalls": 94215, "cacheCreationInputTokens": 24838485, "cacheCreationInputTokensReportedCalls": 31832, "reasoningOutputTokens": 17854947, "reasoningOutputTokensReportedCalls": 100984, "sourceReportedCostSum": 1053.1153779099996, "callsWithReportedCost": 100984 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "diffCheckLikeTurns": 27, "supportedPassedFullTreeCheckTurns": 5 } } }, "cohorts": { "deepseek32": { "trajectories": 500, "issues": 500, "denominator": { "calls": 43683, "callsWithUsage": 43683, "inputTokens": 1177730550, "inputTokensReportedCalls": 43683, "outputTokens": 13441973, "outputTokensReportedCalls": 43683, "totalTokens": 1191172523, "totalTokensReportedCalls": 43683, "cachedInputTokens": 798975703, "cachedInputTokensReportedCalls": 43683, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7941114, "reasoningOutputTokensReportedCalls": 43683, "sourceReportedCostSum": 223.9232482499986, "callsWithReportedCost": 43683 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "diffCheckLikeTurns": 37, "supportedPassedFullTreeCheckTurns": 7 } }, "gemini3flash": { "trajectories": 498, "issues": 498, "denominator": { "calls": 28054, "callsWithUsage": 28054, "inputTokens": 847926935, "inputTokensReportedCalls": 28054, "outputTokens": 10071357, "outputTokensReportedCalls": 28054, "totalTokens": 857998292, "totalTokensReportedCalls": 28054, "cachedInputTokens": 613767718, "cachedInputTokensReportedCalls": 19436, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 7549905, "reasoningOutputTokensReportedCalls": 28054, "sourceReportedCostSum": 177.98206539999973, "callsWithReportedCost": 28054 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "gpt52": { "trajectories": 500, "issues": 500, "denominator": { "calls": 17520, "callsWithUsage": 17520, "inputTokens": 322598043, "inputTokensReportedCalls": 17520, "outputTokens": 6692906, "outputTokensReportedCalls": 17520, "totalTokens": 329290949, "totalTokensReportedCalls": 17520, "cachedInputTokens": 267596032, "cachedInputTokensReportedCalls": 17520, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 4871131, "reasoningOutputTokensReportedCalls": 17520, "sourceReportedCostSum": 236.78350885000094, "callsWithReportedCost": 17520 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "opus45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 16446, "callsWithUsage": 16446, "inputTokens": 330622219, "inputTokensReportedCalls": 16446, "outputTokens": 4806100, "outputTokensReportedCalls": 16446, "totalTokens": 335428319, "totalTokensReportedCalls": 16446, "cachedInputTokens": 314095302, "cachedInputTokensReportedCalls": 16446, "cacheCreationInputTokens": 13695410, "cacheCreationInputTokensReportedCalls": 16446, "reasoningOutputTokens": 1193410, "reasoningOutputTokensReportedCalls": 16446, "sourceReportedCostSum": 376.95399849999967, "callsWithReportedCost": 16446 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": {} }, "sonnet45": { "trajectories": 500, "issues": 500, "denominator": { "calls": 24143, "callsWithUsage": 24143, "inputTokens": 522347941, "inputTokensReportedCalls": 24143, "outputTokens": 7365272, "outputTokensReportedCalls": 24143, "totalTokens": 529713213, "totalTokensReportedCalls": 24143, "cachedInputTokens": 504431119, "cachedInputTokensReportedCalls": 24143, "cacheCreationInputTokens": 17852490, "cacheCreationInputTokensReportedCalls": 24143, "reasoningOutputTokens": 1520265, "reasoningOutputTokensReportedCalls": 24143, "sourceReportedCostSum": 328.9482491999996, "callsWithReportedCost": 24143 }, "proposals": 0, "matchedCalls": 0, "matchedIssues": 0, "matchedUsage": { "calls": 0, "callsWithUsage": 0, "inputTokens": 0, "inputTokensReportedCalls": 0, "outputTokens": 0, "outputTokensReportedCalls": 0, "totalTokens": 0, "totalTokensReportedCalls": 0, "cachedInputTokens": 0, "cachedInputTokensReportedCalls": 0, "cacheCreationInputTokens": 0, "cacheCreationInputTokensReportedCalls": 0, "reasoningOutputTokens": 0, "reasoningOutputTokensReportedCalls": 0, "sourceReportedCostSum": 0, "callsWithReportedCost": 0 }, "matchPercent": null, "callPercent": 0.0, "tokenPercent": 0.0, "deploymentCreditCalls": 0, "deploymentCreditTokens": 0, "matchesWithReasoningOrProse": 0, "diagnostics": { "diffCheckLikeTurns": 3 } } }, "split": { "method": "SHA256('autotoken-v5:'+issue)%5==0 discovery; remaining validation; issue grouped across models", "policyFrozenBeforeValidation": true, "notUntouchedExternalCorpus": true }, "sourceManifestSha256": "73cccd8079f5c3150015207a7d102f1a4409c7996f0ea9ae1305f1b45239247b", "plannerSha256": "797ed0b7305b8ccf1577b3cd510ebf7d8ee2d6cf0f6c9c4271773a8463df33bf", "limitations": [ "Historical explicit completion opt-in and required-test registration are absent: deployment credit is zero even when a conditional syntax match exists.", "The scenario treats the exact prior pytest argv as hypothetically registered required verification, never a registration inferred for deployment.", "A positive pytest summary alone never proposes completion: a later successful full-tree diff check is required within four turns and only supported read-only operations can intervene.", "Pipelines, partial-file checks, redirects other than stderr merge, unknown/mutating actions, failed tests, missing results and stale verification abstain.", "Historical check does not prove staged/untracked changes are approved or that the test selection is sufficient. Runtime policy must register these conditions.", "Exact singleton native submission is the primary label; final prose and submission bundles are not equivalent.", "Submission may contain needed reasoning; removing it can prematurely finish an incomplete task.", "Matched original usage is a conditional gross opportunity, not validated savings. Router, policy registration, reasoning loss and changed future context/cache costs are excluded.", "Cache/reasoning counts are subsets of input/output totals; no double counting.", "No recorded commands executed, tasks submitted or paid inference used." ] }