{
  "schema": "neolabs.v1",
  "dataVersion": "2026-11-18.1",
  "updatedAt": "2026-11-18T15:00:00Z",
  "statuses": {
    "draft": "Prepared from the platform's records; not checked; never rendered on a public page.",
    "validated": "Every figure checked against the Model Lab report and every proof file checked (hashes, ledger entry, time-stamp token); ready, not yet public.",
    "published": "Rendered on the public NEOLABS pages of lilineo.com and neoai.myneogroup.com, in the archive and the sitemap."
  },
  "program": {
    "name": "NEOLABS",
    "title": "Official LILI Capability Tests",
    "operator": "NEO AI - My NEO Group",
    "technology": "NEO TECH LABS TECHNOLOGY LLC",
    "canonicalOrigin": "https://lilineo.com"
  },
  "tests": [
    {
      "id": "test-2",
      "number": 2,
      "status": "published",
      "statusHistory": [
        {
          "status": "draft",
          "at": "2026-11-18T13:40:00Z",
          "note": "Figures read from the Model Lab report of run 01a1ee32-72a5-75b4-87dd-59e9a4be478c (Global Super Admin)."
        },
        {
          "status": "validated",
          "at": "2026-11-18T14:30:00Z",
          "note": "Ledger entry no. 1285 recomputed (MATCH); RFC 3161 token verified with OpenSSL against the DigiCert Trusted Root G4 (Verification: OK); figures cross-checked with the source report."
        },
        {
          "status": "published",
          "at": "2026-11-18T15:00:00Z",
          "note": "Published on the Founder's instruction of 2026-11-18 (NEOLABS prompt and addendum)."
        }
      ],
      "title": "Official Test No. 2",
      "date": "2026-11-18",
      "dateLabel": "18 November 2026",
      "headline": "34 analysis tasks · 5 systems · 170 completed answers",
      "summary": "LILI and two leading single models answered the same 34 analysis tasks through real calls at real prices. Every answer was timed and priced by the platform, scored blind by two independent AI reviewers, sealed in the Proof Ledger and time-stamped by DigiCert in the second the test ended.",
      "labRunId": "01a1ee32-72a5-75b4-87dd-59e9a4be478c",
      "taskSet": {
        "version": "v4-2026-11-17",
        "tasks": 34,
        "languages": [
          "English",
          "French"
        ],
        "kinds": "decision memos, risk assessments and reviews of evidence",
        "frozen": true
      },
      "run": {
        "startedAt": "2026-11-18T10:27:25.989Z",
        "finishedAt": "2026-11-18T12:10:44.196Z",
        "answers": 170,
        "answersCompleted": 170,
        "judgements": 396,
        "spentUsd": 10.36
      },
      "systems": [
        {
          "id": "auto",
          "name": "NEO AUTO",
          "kind": "lili",
          "role": "LILI, balanced profile: a first answer, an independent verification, and a stronger model only when needed.",
          "called": "Through LILI",
          "bestFor": "Demanding work where quality must hold",
          "color": "#7fe0ff"
        },
        {
          "id": "pulse",
          "name": "NEO PULSE",
          "kind": "lili",
          "role": "LILI, speed profile: the fastest route to an answer.",
          "called": "Through LILI",
          "bestFor": "Instant replies, simple requests",
          "color": "#71e3b2"
        },
        {
          "id": "prime",
          "name": "NEO PRIME",
          "kind": "lili",
          "role": "LILI, economy profile: the lowest cost per answer.",
          "called": "Through LILI",
          "bestFor": "Very high volume at minimal cost",
          "color": "#b79aff"
        },
        {
          "id": "claude",
          "name": "Claude Opus 5.5",
          "kind": "reference",
          "role": "Reference: a leading single model.",
          "called": "One call, no orchestration",
          "bestFor": "Reference",
          "color": "#e3c886"
        },
        {
          "id": "gpt",
          "name": "GPT-5.5",
          "kind": "reference",
          "role": "Reference: a leading single model.",
          "called": "One call, no orchestration",
          "bestFor": "Reference",
          "color": "#ff9f8a"
        }
      ],
      "results": {
        "auto": {
          "quality": {
            "mean": 8.703,
            "low": 8.297,
            "high": 9.031,
            "n": 32
          },
          "timeMs": {
            "median": 19004,
            "low": 16488,
            "high": 23159
          },
          "costUsd": {
            "mean": 0.018197,
            "low": 0.006748,
            "high": 0.032491
          }
        },
        "pulse": {
          "quality": {
            "mean": 6.258,
            "low": 5.636,
            "high": 6.864,
            "n": 33
          },
          "timeMs": {
            "median": 1636.5,
            "low": 1556.5,
            "high": 1775
          },
          "costUsd": {
            "mean": 0.011561,
            "low": 0.007497,
            "high": 0.016982
          }
        },
        "prime": {
          "quality": {
            "mean": 5.318,
            "low": 4.727,
            "high": 5.939,
            "n": 33
          },
          "timeMs": {
            "median": 2761.5,
            "low": 2387.5,
            "high": 3368
          },
          "costUsd": {
            "mean": 0.001803,
            "low": 0.001226,
            "high": 0.002497
          }
        },
        "claude": {
          "quality": {
            "mean": 8.906,
            "low": 8.5625,
            "high": 9.203,
            "n": 32
          },
          "timeMs": {
            "median": 32858.5,
            "low": 28750,
            "high": 37160
          },
          "costUsd": {
            "mean": 0.089612,
            "low": 0.073145,
            "high": 0.111604
          }
        },
        "gpt": {
          "quality": {
            "mean": 7.818,
            "low": 7.258,
            "high": 8.348,
            "n": 33
          },
          "timeMs": {
            "median": 26233,
            "low": 19156,
            "high": 34126
          },
          "costUsd": {
            "mean": 0.08413,
            "low": 0.068795,
            "high": 0.101578
          }
        }
      },
      "headlineTable": {
        "title": "NEO AUTO against the strongest reference",
        "rows": [
          {
            "measure": "Cost per answer",
            "auto": "0.018 USD",
            "claude": "0.090 USD",
            "reading": "4.9× lower cost"
          },
          {
            "measure": "Time to a complete answer (median)",
            "auto": "19.0 s",
            "claude": "32.9 s",
            "reading": "1.7× shorter time"
          },
          {
            "measure": "Judged quality (out of 10)",
            "auto": "8.70",
            "claude": "8.91",
            "reading": "No statistically significant difference detected in judged quality on this test."
          }
        ]
      },
      "keyFigures": [
        {
          "value": "4.9×",
          "label": "lower cost per answer for NEO AUTO than Claude Opus 5.5 (0.018 vs 0.090 USD)"
        },
        {
          "value": "12 / 12",
          "label": "cost and speed comparisons statistically significant, all in LILI's favour"
        },
        {
          "value": "+0.95",
          "label": "points out of 10: NEO AUTO judged better than GPT-5.5 (95% interval 0.28 to 1.63; n = 32 paired tasks)"
        },
        {
          "value": "12:10:44 UTC",
          "label": "the sealed results time-stamped by DigiCert, in the second the test ended"
        }
      ],
      "ratios": {
        "note": "Ratios of the average cost and of the median time, rounded. All twelve underlying differences are statistically significant.",
        "rows": [
          {
            "system": "auto",
            "costVsClaude": "4.9×",
            "costVsGpt": "4.6×",
            "timeVsClaude": "1.7×",
            "timeVsGpt": "1.4×"
          },
          {
            "system": "pulse",
            "costVsClaude": "7.8×",
            "costVsGpt": "7.3×",
            "timeVsClaude": "20×",
            "timeVsGpt": "16×"
          },
          {
            "system": "prime",
            "costVsClaude": "50×",
            "costVsGpt": "47×",
            "timeVsClaude": "12×",
            "timeVsGpt": "9.5×"
          }
        ]
      },
      "comparisons": {
        "total": 24,
        "significant": 17,
        "inLiliFavour": 13,
        "notShown": 7,
        "notShownNote": "Seven comparisons are not shown: NEO AUTO against Claude Opus 5.5 on judged quality (no statistically significant difference detected in judged quality on this test) and the six time-to-first-text comparisons, left for a coming test once every profile streams its answers. Under NEOLABS rules a result that is not significant is never reported as a tie or a win.",
        "rules": [
          "A comparison pairs one LILI profile with one reference model on the tasks both answered.",
          "Paired sign-flip test, 20,000 flips, seeded; significant at 0.05 after Holm's correction over all 24 comparisons of the run.",
          "At least 10 paired tasks; the size is the mean paired difference with its 95% bootstrap interval.",
          "Anything else is reported as not shown, never as a tie or a win."
        ],
        "items": [
          {
            "profile": "auto",
            "reference": "claude",
            "measure": "time",
            "diff": -10.73,
            "low": -14.01,
            "high": -7.63,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "auto",
            "reference": "claude",
            "measure": "cost",
            "diff": -0.0714,
            "low": -0.0967,
            "high": -0.0506,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "auto",
            "reference": "gpt",
            "measure": "quality",
            "diff": 0.95,
            "low": 0.28,
            "high": 1.63,
            "n": 32,
            "p": 0.007,
            "favours": "lili"
          },
          {
            "profile": "auto",
            "reference": "gpt",
            "measure": "time",
            "diff": -8.77,
            "low": -14,
            "high": -3.58,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "auto",
            "reference": "gpt",
            "measure": "cost",
            "diff": -0.0659,
            "low": -0.0839,
            "high": -0.0488,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "prime",
            "reference": "claude",
            "measure": "quality",
            "diff": -3.64,
            "low": -4.41,
            "high": -2.91,
            "n": 32,
            "favours": "reference"
          },
          {
            "profile": "prime",
            "reference": "claude",
            "measure": "time",
            "diff": -26.12,
            "low": -29.68,
            "high": -22.6,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "prime",
            "reference": "claude",
            "measure": "cost",
            "diff": -0.0878,
            "low": -0.1102,
            "high": -0.0716,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "prime",
            "reference": "gpt",
            "measure": "quality",
            "diff": -2.5,
            "low": -3.12,
            "high": -1.89,
            "n": 33,
            "favours": "reference"
          },
          {
            "profile": "prime",
            "reference": "gpt",
            "measure": "time",
            "diff": -24.17,
            "low": -31.02,
            "high": -17.95,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "prime",
            "reference": "gpt",
            "measure": "cost",
            "diff": -0.0823,
            "low": -0.1001,
            "high": -0.0664,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "pulse",
            "reference": "claude",
            "measure": "quality",
            "diff": -2.72,
            "low": -3.44,
            "high": -1.92,
            "n": 32,
            "favours": "reference"
          },
          {
            "profile": "pulse",
            "reference": "claude",
            "measure": "time",
            "diff": -29.14,
            "low": -32.93,
            "high": -25.48,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "pulse",
            "reference": "claude",
            "measure": "cost",
            "diff": -0.0781,
            "low": -0.0972,
            "high": -0.0624,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "pulse",
            "reference": "gpt",
            "measure": "quality",
            "diff": -1.56,
            "low": -2.19,
            "high": -0.89,
            "n": 32,
            "favours": "reference"
          },
          {
            "profile": "pulse",
            "reference": "gpt",
            "measure": "time",
            "diff": -27.18,
            "low": -33.19,
            "high": -21.99,
            "n": 34,
            "favours": "lili"
          },
          {
            "profile": "pulse",
            "reference": "gpt",
            "measure": "cost",
            "diff": -0.0726,
            "low": -0.0892,
            "high": -0.0576,
            "n": 34,
            "favours": "lili"
          }
        ]
      },
      "judging": {
        "reviewers": "Two independent AI models, from two providers that wrote none of the answers, scored each anonymised answer out of 10 without knowing its author.",
        "asked": 340,
        "scored": 281,
        "scoredShare": "83%",
        "answers": 170,
        "answersScored": 163,
        "answersWithTwoScores": 118,
        "tasksAllSystemsScored": 32,
        "agreement": {
          "pairs": 118,
          "meanAbsoluteDifference": 1.04,
          "withinOnePoint": "76.3%",
          "pearson": 0.718
        },
        "positionCheck": {
          "pairs": 32,
          "withinOnePoint": "78.1%"
        }
      },
      "layers": {
        "activeCount": 4,
        "items": [
          {
            "n": 1,
            "name": "Dynamic Capability Routing",
            "active": true,
            "text": "Reads each request and selects the tier, the tools and the data it needs, before any model is paid.",
            "evidence": "34 requests routed: 32 to the fast tier first, 2 directly to the top tier"
          },
          {
            "n": 2,
            "name": "Adaptive Model Cascade",
            "active": true,
            "text": "A fast, low-cost model answers first. A top-tier model is called only when the answer needs it.",
            "evidence": "32 first answers by the fast tier; 2 escalated to the top tier"
          },
          {
            "n": 3,
            "name": "Verification-Driven Compute",
            "active": true,
            "text": "An independent model from another provider scores the answer. Evidence decides whether more compute is spent.",
            "evidence": "32 answers verified and accepted"
          },
          {
            "n": 4,
            "name": "Memory and Context Optimization",
            "active": true,
            "text": "LILI sends the facts that matter from its memory graph and a running summary, instead of the whole history.",
            "evidence": "34 requests answered with graph facts"
          },
          {
            "n": 5,
            "name": "Mixture of Agents",
            "active": false,
            "text": "For expert work: up to three models from different providers answer in parallel, then one synthesis and one critic."
          },
          {
            "n": 6,
            "name": "Speculative Parallel Intelligence",
            "active": false,
            "text": "Two approaches are sketched in parallel and scored blind; the stronger one is developed."
          },
          {
            "n": 7,
            "name": "Semantic Compute Reuse",
            "active": false,
            "text": "A verified answer to an equivalent question is reused for the same person instead of being computed again."
          },
          {
            "n": 8,
            "name": "Hybrid Execution",
            "active": false,
            "text": "Designed to place models on API, private or self-hosted capacity. Today every model runs through APIs."
          }
        ],
        "note": "Layers 5 to 8 are built and used by other profiles and request types; they were not at work on this test."
      },
      "decisions": {
        "title": "What LILI decided, across the 34 tasks (NEO AUTO)",
        "rows": [
          {
            "id": "accepted",
            "label": "First answer by a fast-tier model, verified and accepted; top-tier call avoided",
            "tasks": 30,
            "typicalUsd": 0.0027,
            "totalUsd": 0.123
          },
          {
            "id": "escalated",
            "label": "First answer judged insufficient by LILI; escalated to the top-tier model",
            "tasks": 2,
            "typicalUsd": 0.141,
            "totalUsd": 0.281
          },
          {
            "id": "direct",
            "label": "Request routed directly to the top-tier model, then verified",
            "tasks": 2,
            "typicalUsd": 0.107,
            "totalUsd": 0.215
          }
        ],
        "totals": {
          "autoUsd": 0.619,
          "claudeUsd": 3.047
        },
        "reading": "The saving comes from calls that were not made: on 30 of 34 tasks LILI established by verification that a fast-tier answer was good enough. Every escalation is counted in the averages: the 4.9× advantage is net of them."
      },
      "anatomy": {
        "task": "Task 19",
        "request": "A founder wants one assistant that remembers users across five different products. Give the strongest argument for a single shared memory, the strongest argument for keeping each product's memory separate, and the design you would recommend, with the consent and deletion rules it needs.",
        "steps": [
          {
            "name": "Request arrives",
            "text": "The question is received with its context. Nothing is sent to a model yet.",
            "cost": "0 USD"
          },
          {
            "name": "Dynamic Capability Routing",
            "text": "LILI classifies the request and selects a fast-tier model for the first answer.",
            "cost": "0 USD - no model call"
          },
          {
            "name": "First answer",
            "text": "The fast-tier model writes the full answer: 2,054 output tokens.",
            "cost": "0.00156 USD"
          },
          {
            "name": "Verification",
            "text": "An independent model from a different provider scores the answer: 1.0 out of 1.0, accepted.",
            "cost": "0.00056 USD"
          },
          {
            "name": "Adaptive Model Cascade",
            "text": "The cascade was armed. Because the answer was accepted, the top-tier model was not called.",
            "cost": "call avoided"
          }
        ],
        "outcomes": [
          {
            "system": "auto",
            "calls": 2,
            "timeS": 13.6,
            "costUsd": 0.00212,
            "judged": 9
          },
          {
            "system": "claude",
            "calls": 1,
            "timeS": 30.8,
            "costUsd": 0.06315,
            "judged": 9
          },
          {
            "system": "gpt",
            "calls": 1,
            "timeS": 34.1,
            "costUsd": 0.08114,
            "judged": 7
          }
        ],
        "reading": "About 30× lower cost than Claude Opus 5.5 on this request, for the same judged score (9.0).",
        "ledgerEntry": 1143
      },
      "timeline": [
        {
          "at": "2026-11-17",
          "label": "Task set v4 frozen",
          "text": "34 original analysis tasks version-locked before the test."
        },
        {
          "at": "2026-11-18T10:27:25Z",
          "label": "Campaign starts",
          "text": "The first of 170 live calls leaves the Model Lab."
        },
        {
          "at": "2026-11-18T12:10:44.196Z",
          "label": "Campaign ends",
          "text": "170 answers and 396 judgements stored; their fingerprint is computed."
        },
        {
          "at": "2026-11-18T12:10:44.240Z",
          "label": "Sealed in the Proof Ledger",
          "text": "Entry no. 1285, chained to the 1,284 entries before it."
        },
        {
          "at": "2026-11-18T12:10:44Z",
          "label": "Time-stamped by DigiCert",
          "text": "RFC 3161 token, serial 5da7bc65955b06f9c5efc6871bbf4a10. Only the 32-byte hash left NEO AI."
        },
        {
          "at": "2026-11-18",
          "label": "Published by NEOLABS",
          "text": "Methodology, results and proof files made public."
        }
      ],
      "proof": {
        "ledger": {
          "entry": 1285,
          "id": "01a1ee91-0690-7028-9fbb-8501225f1218",
          "at": "2026-11-18T12:10:44.240Z",
          "kind": "lab.campaign",
          "prevHash": "cdf86806c642e1438b1f34cc5987022e8b20301a9bf3ab61be773894ce1cc756",
          "hash": "b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2",
          "resultsDigest": "fb70d29bf897f490f7e131efa6bf54a3b80ee30fde3d8693c5a59682e23a8c5b",
          "rule": "hash = SHA-256( prev_hash || canonical({id, at, kind, account_id, actor_id, run_id, conversation_id, message_id, payload}) ); canonical = JSON with object keys sorted and no spaces."
        },
        "timestamp": {
          "authority": "DigiCert",
          "standard": "RFC 3161",
          "badge": "Results timestamped by DigiCert · RFC 3161",
          "time": "2026-11-18T12:10:44Z",
          "timeLabel": "18 November 2026, 12:10:44 UTC",
          "serial": "5da7bc65955b06f9c5efc6871bbf4a10",
          "policy": "2.16.840.1.114412.7.1",
          "signer": "DigiCert SHA256 RSA4096 Timestamp Responder 2026 1",
          "issuer": "DigiCert Trusted G4 TimeStamping RSA4096 SHA256 2025 CA1",
          "root": "DigiCert Trusted Root G4",
          "rootSha256Fingerprint": "55:2F:7B:DC:F1:A7:AF:9E:6C:E6:72:01:7F:4F:12:AB:F7:72:40:C7:8E:76:1A:C2:03:D1:D9:D2:0A:C8:99:88",
          "digest": "b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2",
          "command": "openssl ts -verify -in test-2-ledger-1285.tsr -digest b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2 -CAfile digicert-trusted-root-g4.pem"
        },
        "files": [
          {
            "name": "ledger-entry-1285.json",
            "label": "Proof Ledger entry no. 1285",
            "sha256": "62a31f1a47a1cb5bdd7fd1ecb6e2c964262807650b1f39d695fb40e48da5bf7d"
          },
          {
            "name": "verify-entry.mjs",
            "label": "Recompute the entry hash (Node.js)",
            "sha256": "3861bdc3ea704b9d29e731f025864359b13f7c552afc982a4e4a6fe84425c384"
          },
          {
            "name": "test-2-ledger-1285.tsr",
            "label": "DigiCert RFC 3161 time-stamp token",
            "sha256": "e9bd2ca9df8fd9b49af122eb887644e09b15fb6244eaf7dfcaa93eff979366b6"
          },
          {
            "name": "digicert-trusted-root-g4.pem",
            "label": "DigiCert Trusted Root G4 (public root certificate)",
            "sha256": "ce7d6b44f5d510391be98c8d76b18709400a30cd87659bfebe1c6f97ff5181ee"
          }
        ],
        "statuses": [
          {
            "id": "timestamp",
            "label": "Third-party timestamp",
            "state": "done",
            "text": "Verified: DigiCert RFC 3161 token over ledger entry no. 1285."
          },
          {
            "id": "audit",
            "label": "External audit",
            "state": "planned",
            "text": "Not yet performed. The test has not been audited by an external party."
          },
          {
            "id": "replication",
            "label": "Independent replication",
            "state": "planned",
            "text": "Not yet performed. The test has not been reproduced by an outside party."
          }
        ],
        "notVerifiablePublicly": "The results digest (fb70d29b...8c5b) covers the 170 answers and 396 judgements; recomputing it needs those records, which are available on request under confidentiality. The ledger entry and the token can be checked by anyone with the files above."
      },
      "caveats": [
        "One run, on one day, on the network of the server: times include the providers' queues and change from day to day.",
        "Quality is judged by AI models, not by people: two reviewers from other providers than the answers, blind to the system; their agreement is reported.",
        "Costs are priced from the tokens reported by each provider and the price table in force on the day.",
        "Some providers do not stream their answers: time to first text is not compared in this test.",
        "Each LILI profile is measured as configured on 18 November 2026; another configuration needs another run.",
        "The time-stamp certifies that the recorded results existed, unchanged, at the stated time. DigiCert provides that public service and has not reviewed this report. The test has not been audited or reproduced by an external party, and results may differ on other tasks, dates or configurations.",
        "Claude Opus 5.5 and GPT-5.5 are products of their respective owners, used as references through their public interfaces."
      ],
      "roadmap": [
        {
          "name": "Breadth",
          "text": "More task families after 34 analytical tasks: reasoning, code, long documents and more languages."
        },
        {
          "name": "Review",
          "text": "Two independent AI reviewers score blind today; human experts will be added on a sample."
        },
        {
          "name": "Precision",
          "text": "Larger samples, to measure the gap with the strongest reference more finely."
        },
        {
          "name": "Independence",
          "text": "Date and integrity are certified by a third party; next, the test reproduced by an outside party."
        }
      ],
      "publicPdf": {
        "file": "neolabs-official-test-2-public-edition.pdf",
        "title": "NEOLABS - Official LILI Capability Test No. 2 - Public Edition"
      }
    }
  ]
}