{"campaign":{"class":"analysis","dates_note":"The dates of execution are declared by NEO AI. The dates attested by third parties are those of the time-stamps: the ledger entry on 18 November 2026 (DigiCert), this pack when it was generated and stamped.","declared_finished_at":"2026-11-18T12:10:44.196Z","declared_started_at":"2026-11-18T10:27:25.989Z","lab_run_id":"01a1ee32-72a5-75b4-87dd-59e9a4be478c","preregistration":{"detail":"Official Test No. 2 ran before the registry of protocols was in place: it was not preregistered. Preregistration starts with Official Test No. 3.","status":"not_applicable"},"public_sample":{"detail":"No public draw was defined before this campaign. The public random sample starts with Official Test No. 3.","status":"not_applicable"},"seed":"cc9f546a26119f0d","systems":5,"task_set_version":"v4-2026-11-17","tasks":34},"corrections":[{"date":"2026-11-20","figures_changed":false,"results_changed":false,"what":"Report text only (Public Edition PDF): the six comparisons not shown besides judged quality are the exact-match comparisons, not time-to-first-text comparisons."}],"costs":{"computed":{"scope":"Inference cost of each answer, computed by the platform from the token counts reported by the providers and the price table in force on the day","unit":"millionths of a US dollar per answer"},"out_of_scope":["Infrastructure and hosting of the platform","Blind review of the answers by the reviewer models"],"reconciled":null,"reconciliation_limit":"Provider invoices and usage reports are issued per organisation and per period, not per campaign: the cost of the answers of this test cannot be attributed from them. The per-answer costs are the platform's computation, not invoices."},"files":[{"bytes":1988,"path":"digicert-trusted-root-g4.pem","role":"DigiCert Trusted Root G4 (public certificate)","sha256":"ce7d6b44f5d510391be98c8d76b18709400a30cd87659bfebe1c6f97ff5181ee"},{"bytes":1944,"path":"ledger-entry-1285.json","role":"Proof Ledger entry no. 1285 (the seal of the campaign)","sha256":"62a31f1a47a1cb5bdd7fd1ecb6e2c964262807650b1f39d695fb40e48da5bf7d"},{"bytes":819117,"path":"neolabs-official-test-2-public-edition.pdf","role":"Report, Public Edition","sha256":"108b44330f0d5a81b566d14bb8f067980720c09e053368943ec15c74a6915779"},{"bytes":47347,"path":"results.json","role":"Per-task results (public rows)","sha256":"5657e55adeb1a5095cc7e641f305a181ed8ccf94a3a93f8347072e7732bea5b3"},{"bytes":6006,"path":"test-2-ledger-1285.tsr","role":"RFC 3161 time-stamp token of the ledger entry (DigiCert)","sha256":"e9bd2ca9df8fd9b49af122eb887644e09b15fb6244eaf7dfcaa93eff979366b6"}],"generated_at":"2026-11-20T08:45:00.000Z","limits":["Quality scores are the judgement of two AI reviewer models, recorded at the time: this pack verifies their aggregation, not the judgement.","The tasks and the answers are not published: the task set stays usable blind.","One class of tasks (analysis), 34 tasks, one day, each LILI profile as configured on that day.","No external party has reproduced or audited this test."],"manifest_version":2,"metrics":{"published":{"claims":[{"difference":{"high":0.3125,"low":-0.71875,"n":32,"value":-0.203125},"measure":"quality","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":0.4898755062246888,"pairs":32,"profile":"profile:auto","supported":false},{"difference":{"high":null,"low":null,"n":0,"value":null},"measure":"exact","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":null,"pairs":0,"profile":"profile:auto","supported":false},{"difference":{"high":-7625.823529411765,"low":-14013.264705882353,"n":34,"value":-10732.029411764706},"measure":"total_ms","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto","supported":true},{"difference":{"high":-50550.91176470588,"low":-96693.88235294117,"n":34,"value":-71414.82352941176},"measure":"cost","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto","supported":true},{"difference":{"high":1.625,"low":0.28125,"n":32,"value":0.953125},"measure":"quality","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":0.007049647517624119,"pairs":32,"profile":"profile:auto","supported":true},{"difference":{"high":null,"low":null,"n":0,"value":null},"measure":"exact","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":null,"pairs":0,"profile":"profile:auto","supported":false},{"difference":{"high":-3583.4117647058824,"low":-13995.85294117647,"n":34,"value":-8773.058823529413},"measure":"total_ms","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":0.0015499225038748063,"pairs":34,"profile":"profile:auto","supported":true},{"difference":{"high":-48836.382352941175,"low":-83896.5,"n":34,"value":-65932.94117647059},"measure":"cost","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto","supported":true},{"difference":{"high":-2.90625,"low":-4.40625,"n":32,"value":-3.640625},"measure":"quality","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":0.00004999750012499375,"pairs":32,"profile":"profile:auto.economy","supported":true},{"difference":{"high":null,"low":null,"n":0,"value":null},"measure":"exact","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":null,"pairs":0,"profile":"profile:auto.economy","supported":false},{"difference":{"high":-22604.41176470588,"low":-29680.882352941175,"n":34,"value":-26124.852941176472},"measure":"total_ms","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto.economy","supported":true},{"difference":{"high":-71622.55882352941,"low":-110193.17647058824,"n":34,"value":-87808.76470588235},"measure":"cost","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto.economy","supported":true},{"difference":{"high":-1.893939393939394,"low":-3.121212121212121,"n":33,"value":-2.5},"measure":"quality","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":0.00004999750012499375,"pairs":33,"profile":"profile:auto.economy","supported":true},{"difference":{"high":null,"low":null,"n":0,"value":null},"measure":"exact","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":null,"pairs":0,"profile":"profile:auto.economy","supported":false},{"difference":{"high":-17948.441176470587,"low":-31020.764705882353,"n":34,"value":-24165.882352941175},"measure":"total_ms","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto.economy","supported":true},{"difference":{"high":-66362.17647058824,"low":-100080.0294117647,"n":34,"value":-82326.88235294117},"measure":"cost","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto.economy","supported":true},{"difference":{"high":-1.921875,"low":-3.4375,"n":32,"value":-2.71875},"measure":"quality","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":0.00004999750012499375,"pairs":32,"profile":"profile:auto.fast","supported":true},{"difference":{"high":null,"low":null,"n":0,"value":null},"measure":"exact","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":null,"pairs":0,"profile":"profile:auto.fast","supported":false},{"difference":{"high":-25484.970588235294,"low":-32925.55882352941,"n":34,"value":-29143.941176470587},"measure":"total_ms","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto.fast","supported":true},{"difference":{"high":-62411.294117647056,"low":-97185.44117647059,"n":34,"value":-78051.44117647059},"measure":"cost","model":"model:model:anthropic/claude/claude-opus-5-5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto.fast","supported":true},{"difference":{"high":-0.890625,"low":-2.1875,"n":32,"value":-1.5625},"measure":"quality","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":0.0002999850007499625,"pairs":32,"profile":"profile:auto.fast","supported":true},{"difference":{"high":null,"low":null,"n":0,"value":null},"measure":"exact","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":null,"pairs":0,"profile":"profile:auto.fast","supported":false},{"difference":{"high":-21988.823529411766,"low":-33190.14705882353,"n":34,"value":-27184.970588235294},"measure":"total_ms","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto.fast","supported":true},{"difference":{"high":-57595.32352941176,"low":-89188.5,"n":34,"value":-72569.55882352941},"measure":"cost","model":"model:model:openai/gpt/gpt-5.5@2026-10-11","p":0.00004999750012499375,"pairs":34,"profile":"profile:auto.fast","supported":true}],"comparisons":24,"judging":{"answers":170,"answers_scored":163,"answers_with_two_scores":118,"mean_absolute_difference":1.0423728813559323,"scores":281,"tasks":34,"tasks_all_systems_scored":32,"within_one_point":0.7627118644067796},"supported":17,"systems":{"model:model:anthropic/claude/claude-opus-5-5@2026-10-11":{"answers":34,"completed":34,"cost_micro_usd":{"high":111603.85294117648,"low":73145.32352941176,"n":34,"value":89612.23529411765},"quality":{"high":9.203125,"low":8.5625,"n":32,"value":8.90625},"total_ms":{"high":37160,"low":28750,"n":34,"value":32858.5}},"model:model:openai/gpt/gpt-5.5@2026-10-11":{"answers":34,"completed":34,"cost_micro_usd":{"high":101577.76470588235,"low":68795.26470588235,"n":34,"value":84130.35294117648},"quality":{"high":8.348484848484848,"low":7.257575757575758,"n":33,"value":7.818181818181818},"total_ms":{"high":34126,"low":19156,"n":34,"value":26233}},"profile:auto":{"answers":34,"completed":34,"cost_micro_usd":{"high":32490.941176470587,"low":6747.558823529412,"n":34,"value":18197.41176470588},"quality":{"high":9.03125,"low":8.296875,"n":32,"value":8.703125},"total_ms":{"high":23159,"low":16488,"n":34,"value":19004}},"profile:auto.economy":{"answers":34,"completed":34,"cost_micro_usd":{"high":2497.1176470588234,"low":1225.5588235294117,"n":34,"value":1803.4705882352941},"quality":{"high":5.9393939393939394,"low":4.7272727272727275,"n":33,"value":5.318181818181818},"total_ms":{"high":3368,"low":2387.5,"n":34,"value":2761.5}},"profile:auto.fast":{"answers":34,"completed":34,"cost_micro_usd":{"high":16981.617647058825,"low":7497.294117647059,"n":34,"value":11560.79411764706},"quality":{"high":6.863636363636363,"low":5.636363636363637,"n":33,"value":6.257575757575758},"total_ms":{"high":1775,"low":1556.5,"n":34,"value":1636.5}}}},"rules":["Judged quality of an answer: the mean of its reviewers' scores (1 to 10). Judged quality of a system: the mean over its completed answers that carry a score; n is that count.","Time: the median total time of the completed answers, in milliseconds. Cost: the mean cost of the completed answers, in millionths of a US dollar, as computed by the platform from the token counts reported by the providers and the price table in force.","Intervals: 95% percentile bootstrap, 2,000 resamples, seeded with the seed of the campaign.","Comparisons: one LILI profile against one reference on the tasks both completed; paired sign-flip test, 20,000 flips, seeded; significant at 0.05 after Holm's correction over all comparisons; at least 10 pairs.","Figures are stored at full precision; a page or a report rounds for display only."],"units":{"cost_micro_usd":"millionths of a US dollar","quality":"points out of 10","total_ms":"milliseconds"}},"previous_manifest_sha256":"a58336b0134bd4387051c185113ee187fdbfe2b28edeae29be7276a82a08cea0","proof_ledger":{"entry":1285,"file":"ledger-entry-1285.json","hash":"b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2","results_digest":"fb70d29bf897f490f7e131efa6bf54a3b80ee30fde3d8693c5a59682e23a8c5b","rule":"hash = SHA-256( prev_hash || canonical({id, at, kind, account_id, actor_id, run_id, conversation_id, message_id, payload}) ); canonical = JSON with object keys sorted and no spaces."},"results":{"by_status":{"completed":170},"exclusions":[],"file":"results.json","merkle":{"construction":"RFC 6962 (SHA-256); leaf = RFC 8785 form of the row","leaves":170,"root":"1dd84c75ab8e74a069633ed43f5aefc3c24c2630a4f3510a96ff6f3061ddad50"},"rows":170,"systems":["model:model:anthropic/claude/claude-opus-5-5@2026-10-11","model:model:openai/gpt/gpt-5.5@2026-10-11","profile:auto","profile:auto.economy","profile:auto.fast"],"tasks":["a01-en","a02-fr","a03-en","a04-fr","a05-en","a06-fr","a07-en","a08-fr","a09-en","a10-fr","a11-en","a12-fr","a13-en","a14-fr","a15-en","a16-fr","a17-en","a18-fr","a19-en","a20-fr","a21-en","a22-fr","a23-en","a24-fr","a25-en","a26-fr","a27-en","a28-fr","a29-en","a30-fr","a31-en","a32-fr","a33-en","a34-fr"]},"schema":"neolabs.manifest/1","test":{"id":"test-2","number":2,"report":"https://lilineo.com/neolabs/official-tests/test-2/","title":"NEOLABS Official Test No. 2"},"timestamps":[{"authority":"DigiCert","covers":"the hash of Proof Ledger entry no. 1285","file":"test-2-ledger-1285.tsr","imprint_sha256":"b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2","serial":"5da7bc65955b06f9c5efc6871bbf4a10","standard":"RFC 3161","time":"2026-11-18T12:10:44Z","verify_command":"openssl ts -verify -in test-2-ledger-1285.tsr -digest b6aec2b0fd46d143fac82b8ba9c63e0edf960084c93b2fbb24456fd2baedb7f2 -CAfile digicert-trusted-root-g4.pem"}],"versions":{"lili":{"commit":"7b2909d","note":"Production revision of the platform during the campaign; each LILI profile as configured on 18 November 2026."},"protocol":"Model Lab protocol of ADR-0080 and ADR-0089: identical tasks, live calls, blind review by two AI models from two providers that wrote none of the answers, seeded statistics.","rubric":"Judged quality from 1 to 10 per answer, rubric of the analysis class.","scorer":{"commit":"7dbd23b","name":"neolabs/verify (score.mjs, stats.mjs)","note":"Published copy of the Model Lab statistics; a test of the platform holds the two identical."},"systems":{"model:model:anthropic/claude/claude-opus-5-5@2026-10-11":"Claude Opus 5.5 (reference)","model:model:openai/gpt/gpt-5.5@2026-10-11":"GPT-5.5 (reference)","profile:auto":"NEO AUTO","profile:auto.economy":"NEO PRIME","profile:auto.fast":"NEO PULSE"}}}