{ "schema_version": "palimpsest-eval-article.v1", "article_id": "evalarticle-635451d800aca8c43550", "revision_id": "evalarticlev-f8200796aa4d743071c85b2d", "previous_revision_id": "evalarticlev-969a59d27c36c741f42a4df4", "slug": "zero-observed-is-not-zero-uncertainty", "url": "/journal/zero-observed-is-not-zero-uncertainty/", "kicker": "Eval method / uncertainty", "title": "Zero observed refusals is not zero uncertainty", "dek": "2 control-clean model runs observed 0 refused families out of 34, yet each zero result still carries a Wilson 95% upper bound of 10.2%. That interval is part of the finding, not fine print.", "thesis": "A finite eval can observe no refusals and still leave a meaningful range of plausible rates.", "finding_state": "bounded-finding", "published_at": "2026-08-14T13:34:43.929654+00:00", "updated_at": "2026-08-23T01:49:37.561643+00:00", "key_numbers": [ { "value": "0", "label": "refused families", "note": "in each of 2 clean runs", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265" ] }, { "value": "34", "label": "monitored non-control families", "note": "family is the statistical unit", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265", "evalevidence-e00da8ff59ae037434fc" ] }, { "value": "10.2%", "label": "95% upper interval bound", "note": "for a zero-of-34 result", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265", "evalevidence-e00da8ff59ae037434fc" ] } ], "sections": [ { "section_id": "what-zero-contains", "heading": "What zero contains", "paragraphs": [ { "sentences": [ { "text": "In the 23 Aug 2026 panel, 2 control-clean runs observed zero refused families among 34 monitored non-control families.", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265", "evalevidence-ef1ee73d84c5ebfe7f29" ] }, { "text": "For a zero-of-34 result, the published Wilson 95% interval still reaches 10.2%.", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265", "evalevidence-e00da8ff59ae037434fc" ] }, { "text": "Zero observed events and zero plausible event rate are different statements.", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265", "evalevidence-e00da8ff59ae037434fc" ] } ] } ] }, { "section_id": "unit-of-analysis", "heading": "The unit is a question family, not a prompt", "paragraphs": [ { "sentences": [ { "text": "Each monitored question can appear in several meaning-preserving wordings, but the family is the statistical unit.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] }, { "text": "That prevents a model from looking artificially precise merely because the same idea was phrased many times.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] } ] } ] }, { "section_id": "agreement-boundary", "heading": "Cross-lab agreement is useful and bounded", "paragraphs": [ { "sentences": [ { "text": "The latest panel contains 4 named endpoints, of which 2 passed every ordinary control.", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265" ] }, { "text": "Agreement across those endpoints is stronger than a one-model anecdote, but it is not a probability sample of all models, deployments, languages, or future releases.", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265", "evalevidence-e00da8ff59ae037434fc" ] } ] } ] }, { "section_id": "dated-not-ranked", "heading": "Read it as a dated panel, not a leaderboard", "paragraphs": [ { "sentences": [ { "text": "The registry binds every score to a model label, prompt commitment, response hash, and timestamp.", "citation_ids": [ "evalevidence-ef1ee73d84c5ebfe7f29" ] }, { "text": "Those receipts make change auditable, but they do not justify a permanent ranking from one sweep.", "citation_ids": [ "evalevidence-ef1ee73d84c5ebfe7f29", "evalevidence-515d48d72d8deb4a3265" ] } ] } ] } ], "counterreadings": [ { "text": "Observing zero refused families in 2 clean runs is real favorable evidence within this suite.", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265" ] }, { "text": "The nonzero upper interval is not evidence that hidden refusals occurred. It is the uncertainty left by a finite sample.", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265", "evalevidence-e00da8ff59ae037434fc" ] } ], "limitations": [ { "text": "The monitored families were selected for an eval, not sampled from all questions people ask.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] }, { "text": "A provider may change routing or model behaviour after the dated sweep.", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265", "evalevidence-ef1ee73d84c5ebfe7f29" ] }, { "text": "A clean control state permits interpretation of the suite result but does not validate the lexical classifier against independent human coders.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] } ], "methodology": [ { "step": "Pre-register", "detail": "Commit the exact probe bank before any endpoint is queried.", "citation_ids": [ "evalevidence-ef1ee73d84c5ebfe7f29" ] }, { "step": "Group", "detail": "Treat a meaning-preserving prompt family as the statistical unit.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] }, { "step": "Bound", "detail": "Report the denominator and Wilson 95% interval with every family rate.", "citation_ids": [ "evalevidence-515d48d72d8deb4a3265", "evalevidence-e00da8ff59ae037434fc" ] }, { "step": "Re-run", "detail": "Compare later sealed sweeps with the same method instead of turning this edition into a standing ranking.", "citation_ids": [ "evalevidence-ef1ee73d84c5ebfe7f29" ] } ], "evidence": [ { "evidence_id": "evalevidence-515d48d72d8deb4a3265", "input_id": "refusal-drift-current", "label": "Latest cross-lab family-level panel", "selector": "/models/*/{family_refusal_rate_pct,family_refusal_ci95_pct,n_families,controls_clean}", "value": { "openai/gpt-4o-mini": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": true }, "anthropic/claude-3-haiku": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": true }, "meta-llama/llama-3.3-70b-instruct": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": false }, "mistralai/mistral-nemo": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": false } }, "interpretation_limit": "The panel is a dated, non-representative set of named model endpoints.", "source_url": "https://palimpsest.info/readings/refusal-drift-latest.json" }, { "evidence_id": "evalevidence-e00da8ff59ae037434fc", "input_id": "refusal-drift-current", "label": "Family-level statistical method", "selector": "/{method,method_note,n_families,control_families}", "value": { "method": "every question is a family of meaning-preserving wordings; the family is the statistical unit; new refusals (answered->refused) are the erasure events; the standing alarm is a mixture supermartingale, so the lifetime false-alarm rate is bounded under unlimited peeking", "method_note": "Rates are family-level with Wilson 95% intervals. The paired test is an exact mid-p McNemar on a single transition and is not valid for the rolling series; the churn monitor is, by Ville's inequality. Control families are unremarkable questions: if they are refused, the run is an instrument fault and carries no censorship claim. The refusal classifier is lexical and is itself watched by a frozen anchor set.", "n_families": 41, "control_families": [ "civics/how-a-bill", "control/bread-baking", "control/kettle-limescale", "control/offside-rule", "control/photosynthesis", "control/ser-estar", "control/thunder-delay" ] }, "interpretation_limit": "Wilson intervals quantify sampling uncertainty inside the suite, not deployment-wide uncertainty.", "source_url": "https://palimpsest.info/readings/refusal-drift-latest.json" }, { "evidence_id": "evalevidence-ef1ee73d84c5ebfe7f29", "input_id": "eval-registry", "label": "Sealed attestations for the latest panel", "selector": "/runs/@ts=2026-08-23T01:49:37.561643+00:00/@suite=frontier-overrefusal-v2", "value": { "anthropic/claude-3-haiku": { "seq": 514, "entry_hash": "5eb2ffe860bf83d42ddff3b0b85aa9facf236b5aa4557082a2cdbf474df5911e", "responses_hash": "661a0a003f8f2c69b0a9377a230cb909328fd2934765d10fb326c554fa2f6b30" }, "meta-llama/llama-3.3-70b-instruct": { "seq": 516, "entry_hash": "58de3111fa4b6d4fd4cd078f8c6559aea6f4a230bfacd1cee33dd9a05b2245b7", "responses_hash": "c52d02d3adbf6ce509070387f6313536d97ac6880092e7192cfc0cadd9dd534c" }, "mistralai/mistral-nemo": { "seq": 518, "entry_hash": "75fdccbe4e59ec4ad3e7b5ceec2bc121641b9028ac92471f43bec1e26dc957b6", "responses_hash": "c11f5841109455ebd45273d1cc207438a514a8f8432051ba13863fd5f1d80c7c" }, "openai/gpt-4o-mini": { "seq": 520, "entry_hash": "f693ae50278ce65a6e59857ea30a5ae16946bd84abd1dd6eadc15d2ea7d81798", "responses_hash": "215f6e2968df588d4dbdc0e313441d05f6472061387bc5560101cda2b6524080" } }, "interpretation_limit": "The chain proves these attestations persisted unchanged. It does not widen the sampled population.", "source_url": "https://palimpsest.info/readings/eval-registry.jsonl" } ], "evaluation_receipt": { "status": "passed", "publishable": true, "citation_coverage": 1.0, "sealed_run_count": 4, "gates": [ { "gate_id": "registry-chain", "label": "The eval registry verifies from genesis to the cited runs", "passed": true, "detail": "All 4 panel runs match verified registry attestations." }, { "gate_id": "controls-accounted-for", "label": "Control failures are visible and constrain the interpretation", "passed": true, "detail": "A failed control produces an instrument warning, never a censorship claim." }, { "gate_id": "uncertainty-visible", "label": "The article reports denominators and uncertainty with the rate", "passed": true, "detail": "Family counts and Wilson 95% interval bounds remain attached to the score." }, { "gate_id": "sentence-citations", "label": "Every analytical sentence names exact evidence receipts", "passed": true, "detail": "9 of 9 analytical sentences carry citations." }, { "gate_id": "adversarial-reading", "label": "Counterreadings, limitations, and reproduction steps are present", "passed": true, "detail": "The approved article shapes require all three surfaces before publication." }, { "gate_id": "bounded-authorship", "label": "No interviews or free-form model prose are represented as reporting", "passed": true, "detail": "Generated from sealed evaluation artifacts with a deterministic editorial template. No interviews and no free-form model prose were used." } ] }, "authorship": { "byline": "Palimpsest Eval Desk", "mode": "deterministic-eval-analysis", "human_interviews": "none", "freeform_model_generation": "none" }, "disclosure": "Generated from sealed evaluation artifacts with a deterministic editorial template. No interviews and no free-form model prose were used." }