{
  "schema": "neolabs.v1",
  "dataVersion": "2026-11-20.3",
  "updatedAt": "2026-11-20T10:00:00Z",
  "statuses": {
    "draft": "Prepared from the platform's records; not checked; never rendered on a public page.",
    "validated": "Every figure checked against the Model Lab report and every proof file checked (hashes, ledger entry, time-stamp token); ready, not yet public.",
    "published": "Rendered on the public NEO AI Evaluations pages of lilineo.com and neoai.myneogroup.com, in the archive and the sitemap."
  },
  "program": {
    "name": "NEO AI Evaluations",
    "tagline": "How we test and measure LILI's capabilities",
    "formerName": "NEOLABS",
    "path": "/evaluations/",
    "promise": "Any change to the registered file can be detected by checking it against its published fingerprint.",
    "register": [
      {
        "id": "proof-ledger",
        "name": "Proof Ledger",
        "kind": "internal",
        "operator": "NEO AI (built-in register; not a third party)",
        "url": null,
        "role": "The hash-chained, append-only register of NEO AI. Each entry contains the fingerprint of the one before it.",
        "confirms": "Proof Ledger entry no. 1285 has not changed since it was sealed, and it is chained to every entry before it. It holds the fingerprint of all 170 answers and 396 judgements of the test.",
        "limit": "Operated by NEO AI: on its own it proves integrity, not a date. The date comes from DigiCert.",
        "statusLabel": "Integrity checkable by anyone",
        "group": "integrity"
      },
      {
        "id": "digicert",
        "name": "DigiCert",
        "kind": "third-party",
        "operator": "DigiCert, Inc. - public certificate authority",
        "url": "https://www.digicert.com/",
        "role": "A public certificate authority that issues RFC 3161 time-stamps.",
        "confirms": "The fingerprint of Proof Ledger entry no. 1285 existed, unchanged, on 18 November 2026 at 12:10:44 UTC.",
        "limit": "DigiCert certifies a date for a fingerprint. It has not reviewed the report, the tasks or the scores.",
        "statusLabel": "Timestamp confirmed",
        "group": "date",
        "source": "ledger-timestamp"
      },
      {
        "id": "freetsa",
        "name": "FreeTSA",
        "kind": "third-party",
        "operator": "FreeTSA (freetsa.org) - free RFC 3161 time-stamp authority",
        "url": "https://freetsa.org/",
        "role": "A free, community-run time-stamp authority (RFC 3161).",
        "confirms": "The manifest of the evidence pack existed at the stated time. The manifest lists the PDF report, the per-task results and the ledger entry with their SHA-256 fingerprints.",
        "limit": "Certifies a date for the manifest. It does not review its content.",
        "statusLabel": "Timestamp confirmed",
        "group": "date",
        "provider": "FreeTSA"
      },
      {
        "id": "sigstore",
        "name": "Sigstore (Fulcio and Rekor)",
        "kind": "third-party",
        "operator": "Sigstore - open-source project of the OpenSSF, Linux Foundation",
        "url": "https://www.sigstore.dev/",
        "role": "Fulcio gives the public GitHub workflow a short-lived signing certificate. Rekor records the signature in a public, append-only transparency log.",
        "confirms": "The manifest was signed by the public workflow of github.com/myneogroup-ai/neolabs-proofs, and that signature is recorded in the public Rekor log.",
        "limit": "Confirms who signed the manifest and that it is unchanged since. It does not judge the figures inside it.",
        "statusLabel": "File integrity verified",
        "group": "integrity",
        "provider": "Sigstore (Fulcio, Rekor)"
      },
      {
        "id": "github-attestation",
        "name": "GitHub Artifact Attestations",
        "kind": "third-party",
        "operator": "GitHub, Inc. (through Sigstore)",
        "url": "https://docs.github.com/en/actions/security-for-github-actions/using-artifact-attestations/using-artifact-attestations-to-establish-provenance-for-builds",
        "verifyUrl": "https://github.com/myneogroup-ai/neolabs-proofs/attestations",
        "role": "Signed build provenance issued by GitHub Actions and recorded in the Rekor log.",
        "confirms": "The published verifier and scorer (verify/*.mjs) come from the public workflow of the neolabs-proofs repository, at a given commit.",
        "limit": "Confirms where the verification code comes from, not that the tested system performs well.",
        "statusLabel": "Provenance attested",
        "group": "provenance",
        "checked": {
          "at": "2026-11-20",
          "how": "GitHub attestations API: 5 provenance attestations for verify/*.mjs, recorded in Rekor (log index 4022392754 for score.mjs)."
        },
        "command": "gh attestation verify verify/score.mjs -R myneogroup-ai/neolabs-proofs",
        "rekorLogIndex": 4022392754
      },
      {
        "id": "opentimestamps",
        "name": "OpenTimestamps (Bitcoin)",
        "kind": "third-party",
        "operator": "OpenTimestamps - open protocol and public calendars, anchored in Bitcoin",
        "url": "https://opentimestamps.org/",
        "role": "A free, open time-stamping protocol anchored in the Bitcoin blockchain, with no company to trust.",
        "confirms": "Once confirmed: the manifest existed before a given Bitcoin block.",
        "limit": "Confirms a date only, with no company to trust. Anyone can check it on their own node with ots verify.",
        "statusLabel": "Timestamp confirmed",
        "group": "date",
        "provider": "OpenTimestamps (Bitcoin)"
      },
      {
        "id": "github-repository",
        "name": "Public proof repository (GitHub)",
        "kind": "host",
        "operator": "Published by NEO TECH LABS TECHNOLOGY L.L.C. on GitHub",
        "url": "https://github.com/myneogroup-ai/neolabs-proofs",
        "role": "Hosts the evidence packs, the verifier and the workflow that signs and time-stamps them, with their full public history. Code under MIT, data under CC BY 4.0.",
        "confirms": "Nothing by itself: it is where anyone can download the files and rerun the checks offline (node verify/cli.mjs verify).",
        "limit": "A host, not a certifier.",
        "statusLabel": "Public",
        "group": "downloads"
      }
    ],
    "planned": [
      {
        "id": "software-heritage",
        "name": "Software Heritage",
        "url": "https://www.softwareheritage.org/",
        "text": "Permanent archive of the public proof repository. Built into the publication workflow; not yet requested."
      },
      {
        "id": "zenodo",
        "name": "Zenodo / DataCite (DOI)",
        "url": "https://zenodo.org/",
        "text": "A permanent identifier (DOI) for each evidence pack. Built into the publication workflow and rehearsed; no permanent deposit yet."
      },
      {
        "id": "hugging-face",
        "name": "Hugging Face",
        "url": "https://huggingface.co/",
        "text": "The per-task public results as an open dataset. Planned, not yet built."
      },
      {
        "id": "osf",
        "name": "OSF preregistration",
        "url": "https://osf.io/",
        "text": "The protocol of a test registered before it runs, from Official Test No. 3. Planned."
      },
      {
        "id": "drand",
        "name": "drand (League of Entropy)",
        "url": "https://drand.love/",
        "text": "Public randomness to draw the audited sample. Built into the publication workflow; used from Official Test No. 3."
      },
      {
        "id": "external-audit",
        "name": "External audit",
        "url": null,
        "text": "Not yet performed. No external party has audited a test."
      },
      {
        "id": "independent-replication",
        "name": "Independent replication",
        "url": null,
        "text": "Not yet performed. No outside party has reproduced a test."
      }
    ],
    "title": "Official LILI Capability Tests",
    "operator": "NEO AI - My NEO Group",
    "technology": "NEO TECH LABS TECHNOLOGY LLC",
    "canonicalOrigin": "https://lilineo.com"
  },
  "tests": [
    {
      "id": "test-2",
      "number": 2,
      "status": "published",
      "statusHistory": [
        {
          "status": "draft",
          "at": "2026-11-18T13:40:00Z",
          "note": "Figures read from the Model Lab report of run 01a1ee32-72a5-75b4-87dd-59e9a4be478c (Global Super Admin)."
        },
        {
          "status": "validated",
          "at": "2026-11-18T14:30:00Z",
          "note": "Ledger entry no. 1285 recomputed (MATCH); RFC 3161 token verified with OpenSSL against the DigiCert Trusted Root G4 (Verification: OK); figures cross-checked with the source report."
        },
        {
          "status": "published",
          "at": "2026-11-18T15:00:00Z",
          "note": "Published on the Founder's instruction of 2026-11-18 (NEOLABS prompt and addendum)."
        }
      ],
      "title": "Official Test No. 2",
      "date": "2026-11-18",
      "dateLabel": "18 November 2026",
      "headline": "34 analysis tasks · 5 systems · 170 completed answers",
      "summary": "LILI and two leading single models answered the same 34 analysis tasks through real calls at real prices. Every answer was timed and priced by the platform, scored blind by two independent AI reviewers, sealed in the Proof Ledger and time-stamped by DigiCert in the second the test ended.",
      "labRunId": "01a1ee32-72a5-75b4-87dd-59e9a4be478c",
      "taskSet": {
        "version": "v4-2026-11-17",
        "tasks": 34,
        "languages": [
          "English",
          "French"
        ],
        "kinds": "decision memos, risk assessments and reviews of evidence",
        "frozen": true
      },
      "run": {
        "startedAt": "2026-11-18T10:27:25.989Z",
        "finishedAt": "2026-11-18T12:10:44.196Z",
        "answers": 170,
        "answersCompleted": 170,
        "judgements": 396,
        "spentUsd": 10.36
      },
      "systems": [
        {
          "id": "auto",
          "name": "NEO AUTO",
          "kind": "lili",
          "role": "LILI, balanced profile: a first answer, an independent verification, and a stronger model only when needed.",
          "called": "Through LILI",
          "bestFor": "Demanding work where quality must hold",
          "color": "#7fe0ff"
        },
        {
          "id": "pulse",
          "name": "NEO PULSE",
          "kind": "lili",
          "role": "LILI, speed profile: the fastest route to an answer.",
          "called": "Through LILI",
          "bestFor": "Instant replies, simple requests",
          "color": "#71e3b2"
        },
        {
          "id": "prime",
          "name": "NEO PRIME",
          "kind": "lili",
          "role": "LILI, economy profile: the lowest cost per answer.",
          "called": "Through LILI",
          "bestFor": "Very high volume at minimal cost",
          "color": "#b79aff"
        },
        {
          "id": "claude",
          "name": "Claude Opus 5.5",
          "kind": "reference",
          "role": "Reference: a leading single model.",
          "called": "One call, no orchestration",
          "bestFor": "Reference",
          "color": "#e3c886"
        },
        {
          "id": "gpt",
          "name": "GPT-5.5",
          "kind": "reference",
          "role": "Reference: a leading single model.",
          "called": "One call, no orchestration",
          "bestFor": "Reference",
          "color": "#ff9f8a"
        }
      ],
      "results": {
        "auto": {
          "quality": {
            "mean": 8.703,
            "low": 8.297,
            "high": 9.031,
            "n": 32
          },
          "timeMs": {
            "median": 19004,
            "low": 16488,
            "high": 23159
          },
          "costUsd": {
            "mean": 0.018197,
            "low": 0.006748,
            "high": 0.032491
          }
        },
        "pulse": {
          "quality": {
            "mean": 6.258,
            "low": 5.636,
            "high": 6.864,
            "n": 33
          },
          "timeMs": {
            "median": 1636.5,
            "low": 1556.5,
            "high": 1775
          },
          "costUsd": {
            "mean": 0.011561,
            "low": 0.007497,
            "high": 0.016982
          }
        },
        "prime": {
          "quality": {
            "mean": 5.318,
            "low": 4.727,
            "high": 5.939,
            "n": 33
          },
          "timeMs": {
            "median": 2761.5,
            "low": 2387.5,
            "high": 3368
          },
          "costUsd": {
            "mean": 0.001803,
            "low": 0.001226,
            "high": 0.002497
          }
        },
        "claude": {
          "quality": {
            "mean": 8.906,
            "low": 8.5625,
            "high": 9.203,
            "n": 32
          },
          "timeMs": {
            "median": 32858.5,
            "low": 28750,
            "high": 37160
          },
          "costUsd": {
            "mean": 0.089612,
            "low": 0.073145,
            "high": 0.111604
          }
        },
        "gpt": {
          "quality": {
            "mean": 7.818,
            "low": 7.258,
            "high": 8.348,
            "n": 33
          },
          "timeMs": {
            "median": 26233,
            "low": 19156,
            "high": 34126
          },
          "costUsd": {
            "mean": 0.08413,
            "low": 0.068795,
            "high": 0.101578
          }
        }
      },
      "headlineTable": {
        "title": "NEO AUTO against the strongest reference",
        "rows": [
          {
            "measure": "Cost per answer",
            "auto": "0.018 USD",
            "claude": "0.090 USD",
            "reading": "4.9× lower cost"
          },
          {
            "measure": "Time to a complete answer (median)",
            "auto": "19.0 s",
            "claude": "32.9 s",
            "reading": "1.7× shorter time"
          },
          {
            "measure": "Judged quality (out of 10)",
            "auto": "8.70",
            "claude": "8.91",
            "reading": "No statistically significant difference detected in judged quality on this test."
          }
        ]
      },
      "keyFigures": [
        {
          "value": "4.9×",
          "label": "lower cost per answer for NEO AUTO than Claude Opus 5.5 (0.018 vs 0.090 USD)"
        },
        {
          "value": "12 / 12",
          "label": "cost and speed comparisons statistically significant, all in LILI's favour"
        },
        {
          "value": "+0.95",
          "label": "points out of 10: NEO AUTO judged better than GPT-5.5 (95% interval 0.28 to 1.63; n = 32 paired tasks)"
        },
        {
          "value": "12:10:44 UTC",
          "label": "the sealed results time-stamped by DigiCert, in the second the test ended"
        }
      ],
      "ratios": {
        "note": "Ratios of the average cost and of the median time, rounded. All twelve underlying differences are statistically significant.",
        "rows": [
          {
            "system": "auto",
            "costVsClaude": "4.9×",
            "costVsGpt": "4.6×",
            "timeVsClaude": "1.7×",
            "timeVsGpt": "1.4×"
          },
          {
            "system": "pulse",
            "costVsClaude": "7.8×",
            "costVsGpt": "7.3×",
            "timeVsClaude": "20×",
            "timeVsGpt": "16×"
          },
          {
            "system": "prime",
            "costVsClaude": "50×",
            "costVsGpt": "47×",
            "timeVsClaude": "12×",
            "timeVsGpt": "9.5×"
          }
        ]
      },
      "comparisons": {
        "total": 24,
        "significant": 17,
        "inLiliFavour": 13,
        "notShown": 7,
        "notShownNote": "Seven comparisons are not shown: NEO AUTO against Claude Opus 5.5 on judged quality (no statistically significant difference detected in judged quality on this test) and the six exact-match comparisons: the analysis tasks of this test have no single known answer, so exact-match accuracy has no paired task to compare. Under the rules of NEO AI Evaluations a result that is not significant is never reported as a tie or a win.",
        "rules": [
          "A comparison pairs one LILI profile with one reference model on the tasks both answered.",
          "Paired sign-flip test, 20,000 flips, seeded; significant at 0.05 after Holm's correction over all 24 comparisons of the run.",
          "At least 10 paired tasks; the size is the mean paired difference with its 95% bootstrap interval.",
          "Anything else is reported as not shown, never as a tie or a win."
        ],
        "items": [
          {
            "profile": "auto",
            "reference": "claude",
            "measure": "time",
            "diff": -10.73,
            "low": -14.01,
            "high": -7.63,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "auto",
            "reference": "claude",
            "measure": "cost",
            "diff": -0.0714,
            "low": -0.0967,
            "high": -0.0506,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "auto",
            "reference": "gpt",
            "measure": "quality",
            "diff": 0.95,
            "low": 0.28,
            "high": 1.63,
            "n": 32,
            "p": 0.007,
            "favours": "lili"
          },
          {
            "profile": "auto",
            "reference": "gpt",
            "measure": "time",
            "diff": -8.77,
            "low": -14,
            "high": -3.58,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "auto",
            "reference": "gpt",
            "measure": "cost",
            "diff": -0.0659,
            "low": -0.0839,
            "high": -0.0488,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "prime",
            "reference": "claude",
            "measure": "quality",
            "diff": -3.64,
            "low": -4.41,
            "high": -2.91,
            "n": 32,
            "favours": "reference"
          },
          {
            "profile": "prime",
            "reference": "claude",
            "measure": "time",
            "diff": -26.12,
            "low": -29.68,
            "high": -22.6,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "prime",
            "reference": "claude",
            "measure": "cost",
            "diff": -0.0878,
            "low": -0.1102,
            "high": -0.0716,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "prime",
            "reference": "gpt",
            "measure": "quality",
            "diff": -2.5,
            "low": -3.12,
            "high": -1.89,
            "n": 33,
            "favours": "reference"
          },
          {
            "profile": "prime",
            "reference": "gpt",
            "measure": "time",
            "diff": -24.17,
            "low": -31.02,
            "high": -17.95,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "prime",
            "reference": "gpt",
            "measure": "cost",
            "diff": -0.0823,
            "low": -0.1001,
            "high": -0.0664,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "pulse",
            "reference": "claude",
            "measure": "quality",
            "diff": -2.72,
            "low": -3.44,
            "high": -1.92,
            "n": 32,
            "favours": "reference"
          },
          {
            "profile": "pulse",
            "reference": "claude",
            "measure": "time",
            "diff": -29.14,
            "low": -32.93,
            "high": -25.48,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "pulse",
            "reference": "claude",
            "measure": "cost",
            "diff": -0.0781,
            "low": -0.0972,
            "high": -0.0624,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "pulse",
            "reference": "gpt",
            "measure": "quality",
            "diff": -1.56,
            "low": -2.19,
            "high": -0.89,
            "n": 32,
            "favours": "reference"
          },
          {
            "profile": "pulse",
            "reference": "gpt",
            "measure": "time",
            "diff": -27.18,
            "low": -33.19,
            "high": -21.99,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "pulse",
            "reference": "gpt",
            "measure": "cost",
            "diff": -0.0726,
            "low": -0.0892,
            "high": -0.0576,
            "n": 34,
            "favours": "lili"
          }
        ]
      },
      "judging": {
        "reviewers": "Two independent AI models, from two providers that wrote none of the answers, scored each anonymised answer out of 10 without knowing its author.",
        "asked": 340,
        "scored": 281,
        "scoredShare": "83%",
        "answers": 170,
        "answersScored": 163,
        "answersWithTwoScores": 118,
        "tasksAllSystemsScored": 32,
        "agreement": {
          "pairs": 118,
          "meanAbsoluteDifference": 1.04,
          "withinOnePoint": "76.3%",
          "pearson": 0.718
        },
        "positionCheck": {
          "pairs": 32,
          "withinOnePoint": "78.1%"
        }
      },
      "layers": {
        "activeCount": 4,
        "items": [
          {
            "n": 1,
            "name": "Dynamic Capability Routing",
            "active": true,
            "text": "Reads each request and selects the tier, the tools and the data it needs, before any model is paid.",
            "evidence": "34 requests routed: 32 to the fast tier first, 2 directly to the top tier"
          },
          {
            "n": 2,
            "name": "Adaptive Model Cascade",
            "active": true,
            "text": "A fast, low-cost model answers first. A top-tier model is called only when the answer needs it.",
            "evidence": "32 first answers by the fast tier; 2 escalated to the top tier"
          },
          {
            "n": 3,
            "name": "Verification-Driven Compute",
            "active": true,
            "text": "An independent model from another provider scores the answer. Evidence decides whether more compute is spent.",
            "evidence": "32 answers verified and accepted"
          },
          {
            "n": 4,
            "name": "Memory and Context Optimization",
            "active": true,
            "text": "LILI sends the facts that matter from its memory graph and a running summary, instead of the whole history.",
            "evidence": "34 requests answered with graph facts"
          },
          {
            "n": 5,
            "name": "Mixture of Agents",
            "active": false,
            "text": "For expert work: up to three models from different providers answer in parallel, then one synthesis and one critic."
          },
          {
            "n": 6,
            "name": "Speculative Parallel Intelligence",
            "active": false,
            "text": "Two approaches are sketched in parallel and scored blind; the stronger one is developed."
          },
          {
            "n": 7,
            "name": "Semantic Compute Reuse",
            "active": false,
            "text": "A verified answer to an equivalent question is reused for the same person instead of being computed again."
          },
          {
            "n": 8,
            "name": "Hybrid Execution",
            "active": false,
            "text": "Designed to place models on API, private or self-hosted capacity. Today every model runs through APIs."
          }
        ],
        "note": "Layers 5 to 8 are built and used by other profiles and request types; they were not at work on this test."
      },
      "decisions": {
        "title": "What LILI decided, across the 34 tasks (NEO AUTO)",
        "rows": [
          {
            "id": "accepted",
            "label": "First answer by a fast-tier model, verified and accepted; top-tier call avoided",
            "tasks": 30,
            "typicalUsd": 0.0027,
            "totalUsd": 0.123
          },
          {
            "id": "escalated",
            "label": "First answer judged insufficient by LILI; escalated to the top-tier model",
            "tasks": 2,
            "typicalUsd": 0.141,
            "totalUsd": 0.281
          },
          {
            "id": "direct",
            "label": "Request routed directly to the top-tier model, then verified",
            "tasks": 2,
            "typicalUsd": 0.107,
            "totalUsd": 0.215
          }
        ],
        "totals": {
          "autoUsd": 0.619,
          "claudeUsd": 3.047
        },
        "reading": "The saving comes from calls that were not made: on 30 of 34 tasks LILI established by verification that a fast-tier answer was good enough. Every escalation is counted in the averages: the 4.9× advantage is net of them."
      },
      "anatomy": {
        "task": "Task 19",
        "request": "A founder wants one assistant that remembers users across five different products. Give the strongest argument for a single shared memory, the strongest argument for keeping each product's memory separate, and the design you would recommend, with the consent and deletion rules it needs.",
        "steps": [
          {
            "name": "Request arrives",
            "text": "The question is received with its context. Nothing is sent to a model yet.",
            "cost": "0 USD"
          },
          {
            "name": "Dynamic Capability Routing",
            "text": "LILI classifies the request and selects a fast-tier model for the first answer.",
            "cost": "0 USD - no model call"
          },
          {
            "name": "First answer",
            "text": "The fast-tier model writes the full answer: 2,054 output tokens.",
            "cost": "0.00156 USD"
          },
          {
            "name": "Verification",
            "text": "An independent model from a different provider scores the answer: 1.0 out of 1.0, accepted.",
            "cost": "0.00056 USD"
          },
          {
            "name": "Adaptive Model Cascade",
            "text": "The cascade was armed. Because the answer was accepted, the top-tier model was not called.",
            "cost": "call avoided"
          }
        ],
        "outcomes": [
          {
            "system": "auto",
            "calls": 2,
            "timeS": 13.6,
            "costUsd": 0.00212,
            "judged": 9
          },
          {
            "system": "claude",
            "calls": 1,
            "timeS": 30.8,
            "costUsd": 0.06315,
            "judged": 9
          },
          {
            "system": "gpt",
            "calls": 1,
            "timeS": 34.1,
            "costUsd": 0.08114,
            "judged": 7
          }
        ],
        "reading": "About 30× lower cost than Claude Opus 5.5 on this request, for the same judged score (9.0).",
        "ledgerEntry": 1143
      },
      "timeline": [
        {
          "at": "2026-11-17",
          "label": "Task set v4 frozen",
          "text": "34 original analysis tasks version-locked before the test."
        },
        {
          "at": "2026-11-18T10:27:25Z",
          "label": "Campaign starts",
          "text": "The first of 170 live calls leaves the Model Lab."
        },
        {
          "at": "2026-11-18T12:10:44.196Z",
          "label": "Campaign ends",
          "text": "170 answers and 396 judgements stored; their fingerprint is computed."
        },
        {
          "at": "2026-11-18T12:10:44.240Z",
          "label": "Sealed in the Proof Ledger",
          "text": "Entry no. 1285, chained to the 1,284 entries before it."
        },
        {
          "at": "2026-11-18T12:10:44Z",
          "label": "Time-stamped by DigiCert",
          "text": "RFC 3161 token, serial 5da7bc65955b06f9c5efc6871bbf4a10. Only the 32-byte hash left NEO AI."
        },
        {
          "at": "2026-11-18",
          "label": "Published",
          "text": "Methodology, results and proof files made public (the programme was then named NEOLABS)."
        },
        {
          "at": "2026-11-20T07:13:55Z",
          "label": "Evidence pack, edition 1",
          "text": "Manifest signed by the public workflow (Sigstore, Rekor) and time-stamped by FreeTSA; scorer provenance attested by GitHub."
        },
        {
          "at": "2026-11-20T08:33:08Z",
          "label": "Evidence pack, edition 2",
          "text": "Report text corrected (comparisons not shown); no figure changed. Signed, logged in Rekor and time-stamped by FreeTSA again; edition 1 stays online."
        }
      ],
      "proof": {
        "ledger": {
          "entry": 1285,
          "id": "01a1ee91-0690-7028-9fbb-8501225f1218",
          "at": "2026-11-18T12:10:44.240Z",
          "kind": "lab.campaign",
          "prevHash": "cdf86806c642e1438b1f34cc5987022e8b20301a9bf3ab61be773894ce1cc756",
          "hash": "b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2",
          "resultsDigest": "fb70d29bf897f490f7e131efa6bf54a3b80ee30fde3d8693c5a59682e23a8c5b",
          "rule": "hash = SHA-256( prev_hash || canonical({id, at, kind, account_id, actor_id, run_id, conversation_id, message_id, payload}) ); canonical = JSON with object keys sorted and no spaces."
        },
        "timestamp": {
          "authority": "DigiCert",
          "standard": "RFC 3161",
          "badge": "Results timestamped by DigiCert · RFC 3161",
          "time": "2026-11-18T12:10:44Z",
          "timeLabel": "18 November 2026, 12:10:44 UTC",
          "serial": "5da7bc65955b06f9c5efc6871bbf4a10",
          "policy": "2.16.840.1.114412.7.1",
          "signer": "DigiCert SHA256 RSA4096 Timestamp Responder 2026 1",
          "issuer": "DigiCert Trusted G4 TimeStamping RSA4096 SHA256 2025 CA1",
          "root": "DigiCert Trusted Root G4",
          "rootSha256Fingerprint": "55:2F:7B:DC:F1:A7:AF:9E:6C:E6:72:01:7F:4F:12:AB:F7:72:40:C7:8E:76:1A:C2:03:D1:D9:D2:0A:C8:99:88",
          "digest": "b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2",
          "command": "openssl ts -verify -in test-2-ledger-1285.tsr -digest b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2 -CAfile digicert-trusted-root-g4.pem"
        },
        "files": [
          {
            "name": "ledger-entry-1285.json",
            "label": "Proof Ledger entry no. 1285",
            "sha256": "62a31f1a47a1cb5bdd7fd1ecb6e2c964262807650b1f39d695fb40e48da5bf7d"
          },
          {
            "name": "verify-entry.mjs",
            "label": "Recompute the entry hash (Node.js)",
            "sha256": "3861bdc3ea704b9d29e731f025864359b13f7c552afc982a4e4a6fe84425c384"
          },
          {
            "name": "test-2-ledger-1285.tsr",
            "label": "DigiCert RFC 3161 time-stamp token",
            "sha256": "e9bd2ca9df8fd9b49af122eb887644e09b15fb6244eaf7dfcaa93eff979366b6"
          },
          {
            "name": "digicert-trusted-root-g4.pem",
            "label": "DigiCert Trusted Root G4 (public root certificate)",
            "sha256": "ce7d6b44f5d510391be98c8d76b18709400a30cd87659bfebe1c6f97ff5181ee"
          }
        ],
        "statuses": [
          {
            "id": "timestamp",
            "label": "Third-party timestamp",
            "state": "done",
            "text": "Verified: DigiCert RFC 3161 token over ledger entry no. 1285."
          },
          {
            "id": "audit",
            "label": "External audit",
            "state": "planned",
            "text": "Not yet performed. The test has not been audited by an external party."
          },
          {
            "id": "replication",
            "label": "Independent replication",
            "state": "planned",
            "text": "Not yet performed. The test has not been reproduced by an outside party."
          }
        ],
        "notVerifiablePublicly": "The results digest (fb70d29b...8c5b) covers the 170 answers and 396 judgements; recomputing it needs those records, which are available on request under confidentiality. The ledger entry and the token can be checked by anyone with the files above."
      },
      "caveats": [
        "One run, on one day, on the network of the server: times include the providers' queues and change from day to day.",
        "Quality is judged by AI models, not by people: two reviewers from other providers than the answers, blind to the system; their agreement is reported.",
        "Costs are priced from the tokens reported by each provider and the price table in force on the day.",
        "Some providers do not stream their answers: time to first text is not compared in this test.",
        "Correction of 20 November 2026: the first edition of this report described the six comparisons not shown, besides judged quality, as time-to-first-text comparisons. They are the exact-match comparisons, which have no paired task on analysis tasks. No figure changed.",
        "Each LILI profile is measured as configured on 18 November 2026; another configuration needs another run.",
        "The time-stamp certifies that the recorded results existed, unchanged, at the stated time. DigiCert provides that public service and has not reviewed this report. The test has not been audited or reproduced by an external party, and results may differ on other tasks, dates or configurations.",
        "Claude Opus 5.5 and GPT-5.5 are products of their respective owners, used as references through their public interfaces."
      ],
      "roadmap": [
        {
          "name": "Breadth",
          "text": "More task families after 34 analytical tasks: reasoning, code, long documents and more languages."
        },
        {
          "name": "Review",
          "text": "Two independent AI reviewers score blind today; human experts will be added on a sample."
        },
        {
          "name": "Precision",
          "text": "Larger samples, to measure the gap with the strongest reference more finely."
        },
        {
          "name": "Independence",
          "text": "Date and integrity are certified by a third party; next, the test reproduced by an outside party."
        }
      ],
      "capabilities": [
        {
          "id": "analysis",
          "scene": "analysis",
          "title": "Reference-level analysis",
          "asked": "34 original analysis tasks - decision memos, risk assessments and reviews of evidence - in English and French. Every system received exactly the same wording.",
          "did": "NEO AUTO, LILI's balanced profile, answered all 34 tasks.",
          "evaluated": "Two AI reviewer models from other providers scored each anonymised answer out of 10, blind to its author.",
          "figure": "8.70 vs 8.91",
          "figureLabel": "judged quality /10, NEO AUTO vs Claude Opus 5.5",
          "conclusion": "No statistically significant difference detected in judged quality on this test. Against GPT-5.5, NEO AUTO was judged better by +0.95 points (95% interval 0.28 to 1.63; n = 32).",
          "limit": "Quality is judged by AI models, on one task family.",
          "anchor": "results"
        },
        {
          "id": "cost",
          "scene": "cost",
          "title": "The same work at a fraction of the cost",
          "asked": "The same 34 tasks, through real calls at real prices.",
          "did": "LILI chose how much compute each request deserved, and called a top-tier model only when needed.",
          "evaluated": "Each call priced from the tokens reported by the provider and the price table of the day, including LILI's verification calls.",
          "figure": "4.9×",
          "figureLabel": "lower cost per answer: 0.018 vs 0.090 USD",
          "conclusion": "Significant on 34 paired tasks after correction for 24 comparisons, net of every escalation.",
          "limit": "Prices of one day; another price table gives other amounts.",
          "anchor": "statistics"
        },
        {
          "id": "speed",
          "scene": "speed",
          "title": "Complete answers sooner",
          "asked": "Time from the request to a complete answer, measured by the server.",
          "did": "NEO AUTO answered with a verified first answer on most tasks instead of a long top-tier generation.",
          "evaluated": "Server timestamps at the start and at the final answer; median over 34 tasks.",
          "figure": "19.0 s vs 32.9 s",
          "figureLabel": "median time to a complete answer, 1.7× shorter",
          "conclusion": "Significant: 10.7 s saved on average per task (95% interval 7.6 to 14.0 s).",
          "limit": "One day, on the server's network: times include the providers' queues.",
          "anchor": "statistics"
        },
        {
          "id": "self-check",
          "scene": "selfcheck",
          "title": "Checks its own work before paying for more",
          "asked": "Answer each task well, without wasting top-tier calls.",
          "did": "On 30 of 34 tasks a fast-tier answer was verified by LILI and accepted. 2 answers were escalated to the top tier; 2 requests went to it directly.",
          "evaluated": "Every decision is a recorded event of the run, counted in the costs and times above.",
          "figure": "30 / 34",
          "figureLabel": "fast answers verified and accepted",
          "conclusion": "The saving comes from the calls LILI did not make, and the escalations are included in the 4.9×.",
          "limit": "A verifier can accept an answer a reviewer later scores lower.",
          "anchor": "decisions"
        },
        {
          "id": "one-answer",
          "scene": "anatomy",
          "title": "One request, opened completely",
          "asked": "Task 19: design one assistant that remembers users across five products, with the consent and deletion rules it needs.",
          "did": "Routed to a fast-tier model, answer verified (1.0 of 1.0), top-tier call avoided: 2 calls, 13.6 s.",
          "evaluated": "Scored 9.0 by the reviewers, the same as Claude Opus 5.5 (9.0), above GPT-5.5 (7.0).",
          "figure": "≈30×",
          "figureLabel": "lower cost on this request: 0.00212 vs 0.06315 USD",
          "conclusion": "Recorded in Proof Ledger entry no. 1143. One request illustrates the mechanism; the averages are the result.",
          "limit": "One task out of 34, shown as an illustration.",
          "anchor": "decisions"
        },
        {
          "id": "profiles",
          "scene": "profiles",
          "title": "A profile for each kind of request",
          "asked": "The same tasks through NEO PULSE (speed) and NEO PRIME (economy).",
          "did": "NEO PULSE answered in a median 1.6 s; NEO PRIME at 0.0018 USD per answer.",
          "evaluated": "Same timing, pricing and blind review as every system.",
          "figure": "20× · 50×",
          "figureLabel": "faster (PULSE) · cheaper (PRIME) than Claude Opus 5.5",
          "conclusion": "On these demanding tasks the references keep the lead on judged quality over PULSE and PRIME: that is why LILI fits the profile to the request.",
          "limit": "Each profile as configured on 18 November 2026.",
          "anchor": "compare"
        }
      ],
      "publicPdf": {
        "file": "neolabs-official-test-2-public-edition.pdf",
        "title": "NEOLABS - Official LILI Capability Test No. 2 - Public Edition",
        "nameNote": "The Public Edition was sealed under the programme's former name, NEOLABS. Its file and fingerprint are unchanged."
      }
    }
  ]
}