{
  "schema": "neolabs.manifest/1",
  "manifest_version": 2,
  "previous_manifest_sha256": "a58336b0134bd4387051c185113ee187fdbfe2b28edeae29be7276a82a08cea0",
  "corrections": [
    {
      "date": "2026-11-20",
      "what": "Report text only (Public Edition PDF): the six comparisons not shown besides judged quality are the exact-match comparisons, not time-to-first-text comparisons.",
      "figures_changed": false,
      "results_changed": false
    }
  ],
  "generated_at": "2026-11-20T08:45:00.000Z",
  "test": {
    "id": "test-2",
    "number": 2,
    "title": "NEOLABS Official Test No. 2",
    "report": "https://lilineo.com/neolabs/official-tests/test-2/"
  },
  "campaign": {
    "lab_run_id": "01a1ee32-72a5-75b4-87dd-59e9a4be478c",
    "declared_started_at": "2026-11-18T10:27:25.989Z",
    "declared_finished_at": "2026-11-18T12:10:44.196Z",
    "dates_note": "The dates of execution are declared by NEO AI. The dates attested by third parties are those of the time-stamps: the ledger entry on 18 November 2026 (DigiCert), this pack when it was generated and stamped.",
    "seed": "cc9f546a26119f0d",
    "task_set_version": "v4-2026-11-17",
    "class": "analysis",
    "tasks": 34,
    "systems": 5,
    "preregistration": {
      "status": "not_applicable",
      "detail": "Official Test No. 2 ran before the registry of protocols was in place: it was not preregistered. Preregistration starts with Official Test No. 3."
    },
    "public_sample": {
      "status": "not_applicable",
      "detail": "No public draw was defined before this campaign. The public random sample starts with Official Test No. 3."
    }
  },
  "versions": {
    "lili": {
      "commit": "7b2909d",
      "note": "Production revision of the platform during the campaign; each LILI profile as configured on 18 November 2026."
    },
    "systems": {
      "profile:auto": "NEO AUTO",
      "profile:auto.economy": "NEO PRIME",
      "profile:auto.fast": "NEO PULSE",
      "model:model:anthropic/claude/claude-opus-5-5@2026-10-11": "Claude Opus 5.5 (reference)",
      "model:model:openai/gpt/gpt-5.5@2026-10-11": "GPT-5.5 (reference)"
    },
    "protocol": "Model Lab protocol of ADR-0080 and ADR-0089: identical tasks, live calls, blind review by two AI models from two providers that wrote none of the answers, seeded statistics.",
    "rubric": "Judged quality from 1 to 10 per answer, rubric of the analysis class.",
    "scorer": {
      "name": "neolabs/verify (score.mjs, stats.mjs)",
      "commit": "7dbd23b",
      "note": "Published copy of the Model Lab statistics; a test of the platform holds the two identical."
    }
  },
  "files": [
    {
      "path": "digicert-trusted-root-g4.pem",
      "role": "DigiCert Trusted Root G4 (public certificate)",
      "bytes": 1988,
      "sha256": "ce7d6b44f5d510391be98c8d76b18709400a30cd87659bfebe1c6f97ff5181ee"
    },
    {
      "path": "ledger-entry-1285.json",
      "role": "Proof Ledger entry no. 1285 (the seal of the campaign)",
      "bytes": 1944,
      "sha256": "62a31f1a47a1cb5bdd7fd1ecb6e2c964262807650b1f39d695fb40e48da5bf7d"
    },
    {
      "path": "neolabs-official-test-2-public-edition.pdf",
      "role": "Report, Public Edition",
      "bytes": 819117,
      "sha256": "108b44330f0d5a81b566d14bb8f067980720c09e053368943ec15c74a6915779"
    },
    {
      "path": "results.json",
      "role": "Per-task results (public rows)",
      "bytes": 47347,
      "sha256": "5657e55adeb1a5095cc7e641f305a181ed8ccf94a3a93f8347072e7732bea5b3"
    },
    {
      "path": "test-2-ledger-1285.tsr",
      "role": "RFC 3161 time-stamp token of the ledger entry (DigiCert)",
      "bytes": 6006,
      "sha256": "e9bd2ca9df8fd9b49af122eb887644e09b15fb6244eaf7dfcaa93eff979366b6"
    }
  ],
  "results": {
    "file": "results.json",
    "rows": 170,
    "by_status": {
      "completed": 170
    },
    "tasks": [
      "a01-en",
      "a02-fr",
      "a03-en",
      "a04-fr",
      "a05-en",
      "a06-fr",
      "a07-en",
      "a08-fr",
      "a09-en",
      "a10-fr",
      "a11-en",
      "a12-fr",
      "a13-en",
      "a14-fr",
      "a15-en",
      "a16-fr",
      "a17-en",
      "a18-fr",
      "a19-en",
      "a20-fr",
      "a21-en",
      "a22-fr",
      "a23-en",
      "a24-fr",
      "a25-en",
      "a26-fr",
      "a27-en",
      "a28-fr",
      "a29-en",
      "a30-fr",
      "a31-en",
      "a32-fr",
      "a33-en",
      "a34-fr"
    ],
    "systems": [
      "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
      "model:model:openai/gpt/gpt-5.5@2026-10-11",
      "profile:auto",
      "profile:auto.economy",
      "profile:auto.fast"
    ],
    "exclusions": [],
    "merkle": {
      "construction": "RFC 6962 (SHA-256); leaf = RFC 8785 form of the row",
      "leaves": 170,
      "root": "1dd84c75ab8e74a069633ed43f5aefc3c24c2630a4f3510a96ff6f3061ddad50"
    }
  },
  "metrics": {
    "rules": [
      "Judged quality of an answer: the mean of its reviewers' scores (1 to 10). Judged quality of a system: the mean over its completed answers that carry a score; n is that count.",
      "Time: the median total time of the completed answers, in milliseconds. Cost: the mean cost of the completed answers, in millionths of a US dollar, as computed by the platform from the token counts reported by the providers and the price table in force.",
      "Intervals: 95% percentile bootstrap, 2,000 resamples, seeded with the seed of the campaign.",
      "Comparisons: one LILI profile against one reference on the tasks both completed; paired sign-flip test, 20,000 flips, seeded; significant at 0.05 after Holm's correction over all comparisons; at least 10 pairs.",
      "Figures are stored at full precision; a page or a report rounds for display only."
    ],
    "units": {
      "quality": "points out of 10",
      "total_ms": "milliseconds",
      "cost_micro_usd": "millionths of a US dollar"
    },
    "published": {
      "systems": {
        "profile:auto": {
          "answers": 34,
          "completed": 34,
          "quality": {
            "value": 8.703125,
            "low": 8.296875,
            "high": 9.03125,
            "n": 32
          },
          "total_ms": {
            "value": 19004,
            "low": 16488,
            "high": 23159,
            "n": 34
          },
          "cost_micro_usd": {
            "value": 18197.41176470588,
            "low": 6747.558823529412,
            "high": 32490.941176470587,
            "n": 34
          }
        },
        "profile:auto.economy": {
          "answers": 34,
          "completed": 34,
          "quality": {
            "value": 5.318181818181818,
            "low": 4.7272727272727275,
            "high": 5.9393939393939394,
            "n": 33
          },
          "total_ms": {
            "value": 2761.5,
            "low": 2387.5,
            "high": 3368,
            "n": 34
          },
          "cost_micro_usd": {
            "value": 1803.4705882352941,
            "low": 1225.5588235294117,
            "high": 2497.1176470588234,
            "n": 34
          }
        },
        "profile:auto.fast": {
          "answers": 34,
          "completed": 34,
          "quality": {
            "value": 6.257575757575758,
            "low": 5.636363636363637,
            "high": 6.863636363636363,
            "n": 33
          },
          "total_ms": {
            "value": 1636.5,
            "low": 1556.5,
            "high": 1775,
            "n": 34
          },
          "cost_micro_usd": {
            "value": 11560.79411764706,
            "low": 7497.294117647059,
            "high": 16981.617647058825,
            "n": 34
          }
        },
        "model:model:anthropic/claude/claude-opus-5-5@2026-10-11": {
          "answers": 34,
          "completed": 34,
          "quality": {
            "value": 8.90625,
            "low": 8.5625,
            "high": 9.203125,
            "n": 32
          },
          "total_ms": {
            "value": 32858.5,
            "low": 28750,
            "high": 37160,
            "n": 34
          },
          "cost_micro_usd": {
            "value": 89612.23529411765,
            "low": 73145.32352941176,
            "high": 111603.85294117648,
            "n": 34
          }
        },
        "model:model:openai/gpt/gpt-5.5@2026-10-11": {
          "answers": 34,
          "completed": 34,
          "quality": {
            "value": 7.818181818181818,
            "low": 7.257575757575758,
            "high": 8.348484848484848,
            "n": 33
          },
          "total_ms": {
            "value": 26233,
            "low": 19156,
            "high": 34126,
            "n": 34
          },
          "cost_micro_usd": {
            "value": 84130.35294117648,
            "low": 68795.26470588235,
            "high": 101577.76470588235,
            "n": 34
          }
        }
      },
      "claims": [
        {
          "measure": "quality",
          "profile": "profile:auto",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 32,
          "difference": {
            "value": -0.203125,
            "low": -0.71875,
            "high": 0.3125,
            "n": 32
          },
          "p": 0.4898755062246888,
          "supported": false
        },
        {
          "measure": "exact",
          "profile": "profile:auto",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 0,
          "difference": {
            "value": null,
            "low": null,
            "high": null,
            "n": 0
          },
          "p": null,
          "supported": false
        },
        {
          "measure": "total_ms",
          "profile": "profile:auto",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -10732.029411764706,
            "low": -14013.264705882353,
            "high": -7625.823529411765,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "cost",
          "profile": "profile:auto",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -71414.82352941176,
            "low": -96693.88235294117,
            "high": -50550.91176470588,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "quality",
          "profile": "profile:auto",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 32,
          "difference": {
            "value": 0.953125,
            "low": 0.28125,
            "high": 1.625,
            "n": 32
          },
          "p": 0.007049647517624119,
          "supported": true
        },
        {
          "measure": "exact",
          "profile": "profile:auto",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 0,
          "difference": {
            "value": null,
            "low": null,
            "high": null,
            "n": 0
          },
          "p": null,
          "supported": false
        },
        {
          "measure": "total_ms",
          "profile": "profile:auto",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -8773.058823529413,
            "low": -13995.85294117647,
            "high": -3583.4117647058824,
            "n": 34
          },
          "p": 0.0015499225038748063,
          "supported": true
        },
        {
          "measure": "cost",
          "profile": "profile:auto",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -65932.94117647059,
            "low": -83896.5,
            "high": -48836.382352941175,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "quality",
          "profile": "profile:auto.economy",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 32,
          "difference": {
            "value": -3.640625,
            "low": -4.40625,
            "high": -2.90625,
            "n": 32
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "exact",
          "profile": "profile:auto.economy",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 0,
          "difference": {
            "value": null,
            "low": null,
            "high": null,
            "n": 0
          },
          "p": null,
          "supported": false
        },
        {
          "measure": "total_ms",
          "profile": "profile:auto.economy",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -26124.852941176472,
            "low": -29680.882352941175,
            "high": -22604.41176470588,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "cost",
          "profile": "profile:auto.economy",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -87808.76470588235,
            "low": -110193.17647058824,
            "high": -71622.55882352941,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "quality",
          "profile": "profile:auto.economy",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 33,
          "difference": {
            "value": -2.5,
            "low": -3.121212121212121,
            "high": -1.893939393939394,
            "n": 33
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "exact",
          "profile": "profile:auto.economy",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 0,
          "difference": {
            "value": null,
            "low": null,
            "high": null,
            "n": 0
          },
          "p": null,
          "supported": false
        },
        {
          "measure": "total_ms",
          "profile": "profile:auto.economy",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -24165.882352941175,
            "low": -31020.764705882353,
            "high": -17948.441176470587,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "cost",
          "profile": "profile:auto.economy",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -82326.88235294117,
            "low": -100080.0294117647,
            "high": -66362.17647058824,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "quality",
          "profile": "profile:auto.fast",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 32,
          "difference": {
            "value": -2.71875,
            "low": -3.4375,
            "high": -1.921875,
            "n": 32
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "exact",
          "profile": "profile:auto.fast",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 0,
          "difference": {
            "value": null,
            "low": null,
            "high": null,
            "n": 0
          },
          "p": null,
          "supported": false
        },
        {
          "measure": "total_ms",
          "profile": "profile:auto.fast",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -29143.941176470587,
            "low": -32925.55882352941,
            "high": -25484.970588235294,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "cost",
          "profile": "profile:auto.fast",
          "model": "model:model:anthropic/claude/claude-opus-5-5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -78051.44117647059,
            "low": -97185.44117647059,
            "high": -62411.294117647056,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "quality",
          "profile": "profile:auto.fast",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 32,
          "difference": {
            "value": -1.5625,
            "low": -2.1875,
            "high": -0.890625,
            "n": 32
          },
          "p": 0.0002999850007499625,
          "supported": true
        },
        {
          "measure": "exact",
          "profile": "profile:auto.fast",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 0,
          "difference": {
            "value": null,
            "low": null,
            "high": null,
            "n": 0
          },
          "p": null,
          "supported": false
        },
        {
          "measure": "total_ms",
          "profile": "profile:auto.fast",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -27184.970588235294,
            "low": -33190.14705882353,
            "high": -21988.823529411766,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        },
        {
          "measure": "cost",
          "profile": "profile:auto.fast",
          "model": "model:model:openai/gpt/gpt-5.5@2026-10-11",
          "pairs": 34,
          "difference": {
            "value": -72569.55882352941,
            "low": -89188.5,
            "high": -57595.32352941176,
            "n": 34
          },
          "p": 0.00004999750012499375,
          "supported": true
        }
      ],
      "comparisons": 24,
      "supported": 17,
      "judging": {
        "answers": 170,
        "answers_scored": 163,
        "answers_with_two_scores": 118,
        "scores": 281,
        "tasks": 34,
        "tasks_all_systems_scored": 32,
        "mean_absolute_difference": 1.0423728813559323,
        "within_one_point": 0.7627118644067796
      }
    }
  },
  "costs": {
    "computed": {
      "scope": "Inference cost of each answer, computed by the platform from the token counts reported by the providers and the price table in force on the day",
      "unit": "millionths of a US dollar per answer"
    },
    "reconciled": null,
    "reconciliation_limit": "Provider invoices and usage reports are issued per organisation and per period, not per campaign: the cost of the answers of this test cannot be attributed from them. The per-answer costs are the platform's computation, not invoices.",
    "out_of_scope": [
      "Infrastructure and hosting of the platform",
      "Blind review of the answers by the reviewer models"
    ]
  },
  "proof_ledger": {
    "entry": 1285,
    "hash": "b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2",
    "results_digest": "fb70d29bf897f490f7e131efa6bf54a3b80ee30fde3d8693c5a59682e23a8c5b",
    "file": "ledger-entry-1285.json",
    "rule": "hash = SHA-256( prev_hash || canonical({id, at, kind, account_id, actor_id, run_id, conversation_id, message_id, payload}) ); canonical = JSON with object keys sorted and no spaces."
  },
  "timestamps": [
    {
      "authority": "DigiCert",
      "standard": "RFC 3161",
      "file": "test-2-ledger-1285.tsr",
      "covers": "the hash of Proof Ledger entry no. 1285",
      "imprint_sha256": "b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2",
      "time": "2026-11-18T12:10:44Z",
      "serial": "5da7bc65955b06f9c5efc6871bbf4a10",
      "verify_command": "openssl ts -verify -in test-2-ledger-1285.tsr -digest b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2 -CAfile digicert-trusted-root-g4.pem"
    }
  ],
  "limits": [
    "Quality scores are the judgement of two AI reviewer models, recorded at the time: this pack verifies their aggregation, not the judgement.",
    "The tasks and the answers are not published: the task set stays usable blind.",
    "One class of tasks (analysis), 34 tasks, one day, each LILI profile as configured on that day.",
    "No external party has reproduced or audited this test."
  ]
}
