{
  "schemaVersion": "controlled-validation-results/v1",
  "protocolCommit": "7ac29403f6ec7fd9cd6890b9d8a125dfe9bcfa9e",
  "matrixCommit": "7ac29403f6ec7fd9cd6890b9d8a125dfe9bcfa9e",
  "protocolSha256": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
  "matrixSha256": "2e54ee5536c0935dd303ce15f53edf4033f1926e681f8e8aca9538ad70553d63",
  "completedRounds": [1, 2, 3],
  "completedRuns": 18,
  "excludedBlocks": ["C"],
  "rawArtifactManifestSha256": "45737099f05e1b3eaf275948a31eb7b8fae209d11f90ec7f7af8961a9db15663",
  "tokenScope": "reportedCumulativeInputTokens is the sum reported by Codex turn events; it is not a context-window size or billing amount.",
  "metricNotes": {
    "repeatedRepositoryScans": "Post-hoc exploratory proxy: literal occurrences of rg, find, git status, or ls in the event log, minus the first occurrence. It is not a structured tool-call count or a causal measure."
  },
  "limitations": [
    "Each cell has three runs on one task; no statistical superiority or causal effect is claimed.",
    "Block C was excluded after its executable validator bytes failed the preregistered commitment.",
    "Post-run scoring corrections are append-only amendments in the public protocol.",
    "Claude Code was not executed; Claude-related statements require external research."
  ],
  "runs": [
    {
      "runId": "r1__ambiguous-issue-2__superpowers",
      "round": 1,
      "taskId": "ambiguous-issue-2",
      "conditionId": "superpowers",
      "attempt": 2,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 1015284.155583,
      "setupDurationMs": 3470.055166,
      "reportedCumulativeInputTokens": 4906389,
      "cachedInputTokens": 4671744,
      "outputTokens": 37064,
      "interactions": 4,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 38,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "d14663d9d5df3e589a2d52795f2fe6367d4f9c67bbcf1bc1ad8fae48cfe2dd9d",
        "diffSha256": "db251d6720c64a5bdc1a935351a932456a7b2a6f8c17507fde5fc0e79ee7c991",
        "promptSha256": "47daff98991f8c939fd4f8ffe67f0d25233399c46799dff5574a679b3626717f"
      }
    },
    {
      "runId": "r1__ambiguous-issue-2__vanilla",
      "round": 1,
      "taskId": "ambiguous-issue-2",
      "conditionId": "vanilla",
      "attempt": 2,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": false,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": false,
      "modelDurationMs": 141238.319209,
      "setupDurationMs": 2937.1066669999996,
      "reportedCumulativeInputTokens": 216519,
      "cachedInputTokens": 191744,
      "outputTokens": 4148,
      "interactions": 1,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 7,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "b868b3cc45f338ef35b82b1ebfbfd6b676accedc4ec686caa7fa7f6dd51bbcfb",
        "diffSha256": "c92ab9f8b4f4d9ac9a65ffd40f493d33993f69d43798ef7ba5a52b2ae88f30f9",
        "promptSha256": "47daff98991f8c939fd4f8ffe67f0d25233399c46799dff5574a679b3626717f"
      }
    },
    {
      "runId": "r1__ambiguous-issue-2__vanilla-explicit",
      "round": 1,
      "taskId": "ambiguous-issue-2",
      "conditionId": "vanilla-explicit",
      "attempt": 1,
      "protocolSha256AtRun": "80f7a3e581f5075b294207a6c784adbd1fe7935db2768984a154e711802f5a60",
      "implementationSuccess": false,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": false,
      "modelDurationMs": 85013.08129100001,
      "setupDurationMs": 2150.3213339999998,
      "reportedCumulativeInputTokens": 165015,
      "cachedInputTokens": 127744,
      "outputTokens": 2349,
      "interactions": 1,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 5,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "732539c8b2579e9cbb038f9afd882519cbbd30dda4bda915ca7dd40e90772bea",
        "diffSha256": "163d21dfa0bbc9734cd75e1f93227c9dab251a548cb9ae353344f5eefbdbb8a9",
        "promptSha256": "47daff98991f8c939fd4f8ffe67f0d25233399c46799dff5574a679b3626717f"
      }
    },
    {
      "runId": "r1__benchmark-2__ponytail",
      "round": 1,
      "taskId": "benchmark-2",
      "conditionId": "ponytail",
      "attempt": 1,
      "protocolSha256AtRun": "80f7a3e581f5075b294207a6c784adbd1fe7935db2768984a154e711802f5a60",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 69973.702125,
      "setupDurationMs": 1138.452542,
      "reportedCumulativeInputTokens": 241446,
      "cachedInputTokens": 200192,
      "outputTokens": 1928,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 12,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "06b8d2a9bdb58558f36191d34401d9ec76d405c80ce829fcd9dac534b715e61a",
        "diffSha256": "a97fa2cafa4e392c90c48e94987a2353fc35a4bfec1cd34cd471e30133028221",
        "promptSha256": "d7dc43ff23320a175218ec1d77a9932190464234e578b606e3b7fa871ace35de"
      }
    },
    {
      "runId": "r1__benchmark-2__vanilla",
      "round": 1,
      "taskId": "benchmark-2",
      "conditionId": "vanilla",
      "attempt": 1,
      "protocolSha256AtRun": "80f7a3e581f5075b294207a6c784adbd1fe7935db2768984a154e711802f5a60",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 92271.772042,
      "setupDurationMs": 103.465542,
      "reportedCumulativeInputTokens": 347851,
      "cachedInputTokens": 323840,
      "outputTokens": 2676,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 16,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "1051864ee63a3a2ac666f305ea803252cfdbdcbe0271820724554a11133b7eee",
        "diffSha256": "b7ea16ccdcfedb5f57563c90152b248c273f1bc788147fee1621665edd8bc82e",
        "promptSha256": "d7dc43ff23320a175218ec1d77a9932190464234e578b606e3b7fa871ace35de"
      }
    },
    {
      "runId": "r1__benchmark-2__vanilla-explicit",
      "round": 1,
      "taskId": "benchmark-2",
      "conditionId": "vanilla-explicit",
      "attempt": 1,
      "protocolSha256AtRun": "80f7a3e581f5075b294207a6c784adbd1fe7935db2768984a154e711802f5a60",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 559697.124959,
      "setupDurationMs": 130.877542,
      "reportedCumulativeInputTokens": 183325,
      "cachedInputTokens": 165632,
      "outputTokens": 1971,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 9,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "1e01411ea4820909278018929936b182dbd6554c36dac338c4b4462f41ab28c9",
        "diffSha256": "a97fa2cafa4e392c90c48e94987a2353fc35a4bfec1cd34cd471e30133028221",
        "promptSha256": "d7dc43ff23320a175218ec1d77a9932190464234e578b606e3b7fa871ace35de"
      }
    },
    {
      "runId": "r2__ambiguous-issue-2__superpowers",
      "round": 2,
      "taskId": "ambiguous-issue-2",
      "conditionId": "superpowers",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": false,
      "operationalCompletion": false,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": false,
      "modelDurationMs": 48672.398166,
      "setupDurationMs": 3327.31425,
      "reportedCumulativeInputTokens": 156560,
      "cachedInputTokens": 118016,
      "outputTokens": 1707,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 8,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "21cf3bdb6f362643dc93f6088a38ec5054ea3a6977f81b43cfc105f5841696e6",
        "diffSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
        "promptSha256": "47daff98991f8c939fd4f8ffe67f0d25233399c46799dff5574a679b3626717f"
      }
    },
    {
      "runId": "r2__ambiguous-issue-2__vanilla",
      "round": 2,
      "taskId": "ambiguous-issue-2",
      "conditionId": "vanilla",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": false,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": false,
      "modelDurationMs": 629383.101292,
      "setupDurationMs": 2753.637084,
      "reportedCumulativeInputTokens": 421349,
      "cachedInputTokens": 373248,
      "outputTokens": 5205,
      "interactions": 1,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 7,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "85a72ce441d75a6cff5999934875717a624a402693206b3a3c19c03ec9609b34",
        "diffSha256": "a67872d9c31e31c788b2dd907df59ab965a0cc14d1a57e166a8d4bedf9748cf1",
        "promptSha256": "47daff98991f8c939fd4f8ffe67f0d25233399c46799dff5574a679b3626717f"
      }
    },
    {
      "runId": "r2__ambiguous-issue-2__vanilla-explicit",
      "round": 2,
      "taskId": "ambiguous-issue-2",
      "conditionId": "vanilla-explicit",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 144003.658751,
      "setupDurationMs": 2979.2998749999997,
      "reportedCumulativeInputTokens": 264380,
      "cachedInputTokens": 224000,
      "outputTokens": 3992,
      "interactions": 1,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 7,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "f92a2b6fae04fbbaf57f698c8177d828c09c8582bd02522c1a5539ee075c617b",
        "diffSha256": "157e4a3d78d554739d1d98e90b79f4987ca09b23efb469a13be639e2b0cc8395",
        "promptSha256": "47daff98991f8c939fd4f8ffe67f0d25233399c46799dff5574a679b3626717f"
      }
    },
    {
      "runId": "r2__benchmark-2__ponytail",
      "round": 2,
      "taskId": "benchmark-2",
      "conditionId": "ponytail",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": true,
      "operationalCompletion": false,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 73669.695916,
      "setupDurationMs": 1433.4208749999998,
      "reportedCumulativeInputTokens": 233617,
      "cachedInputTokens": 195072,
      "outputTokens": 1975,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 10,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "021d8418100ba9da95620a900e68d789230a78a075d74237f1b27c2b316d7781",
        "diffSha256": "a97fa2cafa4e392c90c48e94987a2353fc35a4bfec1cd34cd471e30133028221",
        "promptSha256": "d7dc43ff23320a175218ec1d77a9932190464234e578b606e3b7fa871ace35de"
      }
    },
    {
      "runId": "r2__benchmark-2__vanilla",
      "round": 2,
      "taskId": "benchmark-2",
      "conditionId": "vanilla",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 1196553.033583,
      "setupDurationMs": 132.692458,
      "reportedCumulativeInputTokens": 330715,
      "cachedInputTokens": 283904,
      "outputTokens": 2884,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 10,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "8421bd6ab8e0c93b06ba02c95f869964ca60882fe2b862ad5afcb1dbd0c71655",
        "diffSha256": "b7ea16ccdcfedb5f57563c90152b248c273f1bc788147fee1621665edd8bc82e",
        "promptSha256": "d7dc43ff23320a175218ec1d77a9932190464234e578b606e3b7fa871ace35de"
      }
    },
    {
      "runId": "r2__benchmark-2__vanilla-explicit",
      "round": 2,
      "taskId": "benchmark-2",
      "conditionId": "vanilla-explicit",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 68522.1115,
      "setupDurationMs": 134.9855,
      "reportedCumulativeInputTokens": 182503,
      "cachedInputTokens": 164608,
      "outputTokens": 1702,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 9,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "f03add22b9bb5a98e2019badcb947c16ee433a88f2491b77f09d2dd23843cf70",
        "diffSha256": "60c2fbe6ff0500eaefc0eb7fe21aa84e49c8af343f04b5d21e24787b090d3994",
        "promptSha256": "d7dc43ff23320a175218ec1d77a9932190464234e578b606e3b7fa871ace35de"
      }
    },
    {
      "runId": "r3__ambiguous-issue-2__superpowers",
      "round": 3,
      "taskId": "ambiguous-issue-2",
      "conditionId": "superpowers",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 549418.972751,
      "setupDurationMs": 2685.4636250000003,
      "reportedCumulativeInputTokens": 5539327,
      "cachedInputTokens": 5243136,
      "outputTokens": 54029,
      "interactions": 5,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 35,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "f7d984dd445aa56e029ee65dc090d1c5cfc26f6a74e21ab86409b23df4f215b1",
        "diffSha256": "11d98497e76b13a096971f04da0ced125517c653cc54a443b23cf3fb4132d701",
        "promptSha256": "47daff98991f8c939fd4f8ffe67f0d25233399c46799dff5574a679b3626717f"
      }
    },
    {
      "runId": "r3__ambiguous-issue-2__vanilla",
      "round": 3,
      "taskId": "ambiguous-issue-2",
      "conditionId": "vanilla",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": false,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": false,
      "modelDurationMs": 121979.35454100001,
      "setupDurationMs": 2161.516208,
      "reportedCumulativeInputTokens": 238709,
      "cachedInputTokens": 200704,
      "outputTokens": 3911,
      "interactions": 1,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 5,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "8e50eab70dba335b77bbb0b14cda5b777eaf47f42ea4e369cb8fbf5cf02c9cf6",
        "diffSha256": "fd5dc0cc6885d75644bec37d3562035caa3aad92956c8792fd1c393f9d9acafd",
        "promptSha256": "47daff98991f8c939fd4f8ffe67f0d25233399c46799dff5574a679b3626717f"
      }
    },
    {
      "runId": "r3__ambiguous-issue-2__vanilla-explicit",
      "round": 3,
      "taskId": "ambiguous-issue-2",
      "conditionId": "vanilla-explicit",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": false,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": false,
      "modelDurationMs": 89419.344541,
      "setupDurationMs": 2220.119375,
      "reportedCumulativeInputTokens": 187127,
      "cachedInputTokens": 164352,
      "outputTokens": 2507,
      "interactions": 1,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 5,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "b9214caa97ece88a09063d9b0f766881689e67591a6b135092ba2767210aca84",
        "diffSha256": "791908ce82cc7fbe4b59c13d414a61217931419380a169bd87fdbe1bb65423fe",
        "promptSha256": "47daff98991f8c939fd4f8ffe67f0d25233399c46799dff5574a679b3626717f"
      }
    },
    {
      "runId": "r3__benchmark-2__ponytail",
      "round": 3,
      "taskId": "benchmark-2",
      "conditionId": "ponytail",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 103840.990791,
      "setupDurationMs": 6346.386125,
      "reportedCumulativeInputTokens": 293447,
      "cachedInputTokens": 251904,
      "outputTokens": 2183,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 11,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "8560f3c39bd6236468b37d5a59e302371af914a65915939f0c143f61b17e0aca",
        "diffSha256": "a97fa2cafa4e392c90c48e94987a2353fc35a4bfec1cd34cd471e30133028221",
        "promptSha256": "d7dc43ff23320a175218ec1d77a9932190464234e578b606e3b7fa871ace35de"
      }
    },
    {
      "runId": "r3__benchmark-2__vanilla",
      "round": 3,
      "taskId": "benchmark-2",
      "conditionId": "vanilla",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 75440.80283300001,
      "setupDurationMs": 97.60062500000001,
      "reportedCumulativeInputTokens": 216617,
      "cachedInputTokens": 168448,
      "outputTokens": 2006,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 10,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "d95234ee95a960e41e6dd90f171681267cea0b4a2684488262968c06d1b9182a",
        "diffSha256": "60c2fbe6ff0500eaefc0eb7fe21aa84e49c8af343f04b5d21e24787b090d3994",
        "promptSha256": "d7dc43ff23320a175218ec1d77a9932190464234e578b606e3b7fa871ace35de"
      }
    },
    {
      "runId": "r3__benchmark-2__vanilla-explicit",
      "round": 3,
      "taskId": "benchmark-2",
      "conditionId": "vanilla-explicit",
      "attempt": 1,
      "protocolSha256AtRun": "57f83a9f8efe77d795be9c0d1eea5ae8bc1577790c7335712a521861d0e98dec",
      "implementationSuccess": true,
      "operationalCompletion": true,
      "publicTestsPassed": true,
      "coreHiddenTestsPassed": true,
      "modelDurationMs": 62555.273667,
      "setupDurationMs": 94.28550000000001,
      "reportedCumulativeInputTokens": 157939,
      "cachedInputTokens": 138240,
      "outputTokens": 1775,
      "interactions": 0,
      "agents": {
        "main": 1,
        "subagents": 0
      },
      "repeatedRepositoryScans": 5,
      "leftoverChildProcesses": 0,
      "artifactCommitments": {
        "eventsSha256": "f67ce9382a85c60f259b3de5d161e7a62309ec28674a57da895d2ff41fdda840",
        "diffSha256": "60c2fbe6ff0500eaefc0eb7fe21aa84e49c8af343f04b5d21e24787b090d3994",
        "promptSha256": "d7dc43ff23320a175218ec1d77a9932190464234e578b606e3b7fa871ace35de"
      }
    }
  ],
  "cells": [
    {
      "taskId": "ambiguous-issue-2",
      "conditionId": "superpowers",
      "n": 3,
      "implementationSuccess": "2/3",
      "operationalCompletion": "2/3",
      "modelDurationMs": {
        "raw": [1015284.155583, 48672.398166, 549418.972751],
        "median": 549418.972751,
        "min": 48672.398166,
        "max": 1015284.155583
      },
      "reportedCumulativeInputTokens": {
        "raw": [4906389, 156560, 5539327],
        "median": 4906389,
        "min": 156560,
        "max": 5539327
      },
      "outputTokens": {
        "raw": [37064, 1707, 54029],
        "median": 37064,
        "min": 1707,
        "max": 54029
      },
      "interactions": {
        "raw": [4, 0, 5],
        "median": 4,
        "min": 0,
        "max": 5
      },
      "successfulRuns": {
        "n": 2,
        "modelDurationMs": [1015284.155583, 549418.972751]
      },
      "unsuccessfulRuns": {
        "n": 1,
        "modelDurationMs": [48672.398166]
      }
    },
    {
      "taskId": "ambiguous-issue-2",
      "conditionId": "vanilla",
      "n": 3,
      "implementationSuccess": "0/3",
      "operationalCompletion": "3/3",
      "modelDurationMs": {
        "raw": [141238.319209, 629383.101292, 121979.35454100001],
        "median": 141238.319209,
        "min": 121979.35454100001,
        "max": 629383.101292
      },
      "reportedCumulativeInputTokens": {
        "raw": [216519, 421349, 238709],
        "median": 238709,
        "min": 216519,
        "max": 421349
      },
      "outputTokens": {
        "raw": [4148, 5205, 3911],
        "median": 4148,
        "min": 3911,
        "max": 5205
      },
      "interactions": {
        "raw": [1, 1, 1],
        "median": 1,
        "min": 1,
        "max": 1
      },
      "successfulRuns": {
        "n": 0,
        "modelDurationMs": []
      },
      "unsuccessfulRuns": {
        "n": 3,
        "modelDurationMs": [141238.319209, 629383.101292, 121979.35454100001]
      }
    },
    {
      "taskId": "ambiguous-issue-2",
      "conditionId": "vanilla-explicit",
      "n": 3,
      "implementationSuccess": "1/3",
      "operationalCompletion": "3/3",
      "modelDurationMs": {
        "raw": [85013.08129100001, 144003.658751, 89419.344541],
        "median": 89419.344541,
        "min": 85013.08129100001,
        "max": 144003.658751
      },
      "reportedCumulativeInputTokens": {
        "raw": [165015, 264380, 187127],
        "median": 187127,
        "min": 165015,
        "max": 264380
      },
      "outputTokens": {
        "raw": [2349, 3992, 2507],
        "median": 2507,
        "min": 2349,
        "max": 3992
      },
      "interactions": {
        "raw": [1, 1, 1],
        "median": 1,
        "min": 1,
        "max": 1
      },
      "successfulRuns": {
        "n": 1,
        "modelDurationMs": [144003.658751]
      },
      "unsuccessfulRuns": {
        "n": 2,
        "modelDurationMs": [85013.08129100001, 89419.344541]
      }
    },
    {
      "taskId": "benchmark-2",
      "conditionId": "ponytail",
      "n": 3,
      "implementationSuccess": "3/3",
      "operationalCompletion": "2/3",
      "modelDurationMs": {
        "raw": [69973.702125, 73669.695916, 103840.990791],
        "median": 73669.695916,
        "min": 69973.702125,
        "max": 103840.990791
      },
      "reportedCumulativeInputTokens": {
        "raw": [241446, 233617, 293447],
        "median": 241446,
        "min": 233617,
        "max": 293447
      },
      "outputTokens": {
        "raw": [1928, 1975, 2183],
        "median": 1975,
        "min": 1928,
        "max": 2183
      },
      "interactions": {
        "raw": [0, 0, 0],
        "median": 0,
        "min": 0,
        "max": 0
      },
      "successfulRuns": {
        "n": 3,
        "modelDurationMs": [69973.702125, 73669.695916, 103840.990791]
      },
      "unsuccessfulRuns": {
        "n": 0,
        "modelDurationMs": []
      }
    },
    {
      "taskId": "benchmark-2",
      "conditionId": "vanilla",
      "n": 3,
      "implementationSuccess": "3/3",
      "operationalCompletion": "3/3",
      "modelDurationMs": {
        "raw": [92271.772042, 1196553.033583, 75440.80283300001],
        "median": 92271.772042,
        "min": 75440.80283300001,
        "max": 1196553.033583
      },
      "reportedCumulativeInputTokens": {
        "raw": [347851, 330715, 216617],
        "median": 330715,
        "min": 216617,
        "max": 347851
      },
      "outputTokens": {
        "raw": [2676, 2884, 2006],
        "median": 2676,
        "min": 2006,
        "max": 2884
      },
      "interactions": {
        "raw": [0, 0, 0],
        "median": 0,
        "min": 0,
        "max": 0
      },
      "successfulRuns": {
        "n": 3,
        "modelDurationMs": [92271.772042, 1196553.033583, 75440.80283300001]
      },
      "unsuccessfulRuns": {
        "n": 0,
        "modelDurationMs": []
      }
    },
    {
      "taskId": "benchmark-2",
      "conditionId": "vanilla-explicit",
      "n": 3,
      "implementationSuccess": "3/3",
      "operationalCompletion": "3/3",
      "modelDurationMs": {
        "raw": [559697.124959, 68522.1115, 62555.273667],
        "median": 68522.1115,
        "min": 62555.273667,
        "max": 559697.124959
      },
      "reportedCumulativeInputTokens": {
        "raw": [183325, 182503, 157939],
        "median": 182503,
        "min": 157939,
        "max": 183325
      },
      "outputTokens": {
        "raw": [1971, 1702, 1775],
        "median": 1775,
        "min": 1702,
        "max": 1971
      },
      "interactions": {
        "raw": [0, 0, 0],
        "median": 0,
        "min": 0,
        "max": 0
      },
      "successfulRuns": {
        "n": 3,
        "modelDurationMs": [559697.124959, 68522.1115, 62555.273667]
      },
      "unsuccessfulRuns": {
        "n": 0,
        "modelDurationMs": []
      }
    }
  ]
}