{ "schema_version": "palimpsest-eval-article.v1", "article_id": "evalarticle-635451d800aca8c43550", "revision_id": "evalarticlev-d5cf18bb7c03b0d76ddf9678", "previous_revision_id": "evalarticlev-f8200796aa4d743071c85b2d", "slug": "zero-observed-is-not-zero-uncertainty", "url": "/journal/zero-observed-is-not-zero-uncertainty/", "kicker": "Eval method / uncertainty", "title": "Zero observed refusals is not zero uncertainty", "dek": "3 control-clean model runs observed 0 refused families out of 34, yet each zero result still carries a Wilson 95% upper bound of 10.2%. That interval is part of the finding, not fine print.", "thesis": "A finite eval can observe no refusals and still leave a meaningful range of plausible rates.", "finding_state": "bounded-finding", "published_at": "2026-08-14T13:34:43.929654+00:00", "updated_at": "2026-08-23T06:54:58.055463+00:00", "key_numbers": [ { "value": "0", "label": "refused families", "note": "in each of 3 clean runs", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889" ] }, { "value": "34", "label": "monitored non-control families", "note": "family is the statistical unit", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889", "evalevidence-e00da8ff59ae037434fc" ] }, { "value": "10.2%", "label": "95% upper interval bound", "note": "for a zero-of-34 result", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889", "evalevidence-e00da8ff59ae037434fc" ] } ], "sections": [ { "section_id": "what-zero-contains", "heading": "What zero contains", "paragraphs": [ { "sentences": [ { "text": "In the 23 Aug 2026 panel, 3 control-clean runs observed zero refused families among 34 monitored non-control families.", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889", "evalevidence-0091d719054480ab6242" ] }, { "text": "For a zero-of-34 result, the published Wilson 95% interval still reaches 10.2%.", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889", "evalevidence-e00da8ff59ae037434fc" ] }, { "text": "Zero observed events and zero plausible event rate are different statements.", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889", "evalevidence-e00da8ff59ae037434fc" ] } ] } ] }, { "section_id": "unit-of-analysis", "heading": "The unit is a question family, not a prompt", "paragraphs": [ { "sentences": [ { "text": "Each monitored question can appear in several meaning-preserving wordings, but the family is the statistical unit.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] }, { "text": "That prevents a model from looking artificially precise merely because the same idea was phrased many times.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] } ] } ] }, { "section_id": "agreement-boundary", "heading": "Cross-lab agreement is useful and bounded", "paragraphs": [ { "sentences": [ { "text": "The latest panel contains 4 named endpoints, of which 3 passed every ordinary control.", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889" ] }, { "text": "Agreement across those endpoints is stronger than a one-model anecdote, but it is not a probability sample of all models, deployments, languages, or future releases.", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889", "evalevidence-e00da8ff59ae037434fc" ] } ] } ] }, { "section_id": "dated-not-ranked", "heading": "Read it as a dated panel, not a leaderboard", "paragraphs": [ { "sentences": [ { "text": "The registry binds every score to a model label, prompt commitment, response hash, and timestamp.", "citation_ids": [ "evalevidence-0091d719054480ab6242" ] }, { "text": "Those receipts make change auditable, but they do not justify a permanent ranking from one sweep.", "citation_ids": [ "evalevidence-0091d719054480ab6242", "evalevidence-bfa74cade8e7c59f1889" ] } ] } ] } ], "counterreadings": [ { "text": "Observing zero refused families in 3 clean runs is real favorable evidence within this suite.", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889" ] }, { "text": "The nonzero upper interval is not evidence that hidden refusals occurred. It is the uncertainty left by a finite sample.", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889", "evalevidence-e00da8ff59ae037434fc" ] } ], "limitations": [ { "text": "The monitored families were selected for an eval, not sampled from all questions people ask.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] }, { "text": "A provider may change routing or model behaviour after the dated sweep.", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889", "evalevidence-0091d719054480ab6242" ] }, { "text": "A clean control state permits interpretation of the suite result but does not validate the lexical classifier against independent human coders.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] } ], "methodology": [ { "step": "Pre-register", "detail": "Commit the exact probe bank before any endpoint is queried.", "citation_ids": [ "evalevidence-0091d719054480ab6242" ] }, { "step": "Group", "detail": "Treat a meaning-preserving prompt family as the statistical unit.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] }, { "step": "Bound", "detail": "Report the denominator and Wilson 95% interval with every family rate.", "citation_ids": [ "evalevidence-bfa74cade8e7c59f1889", "evalevidence-e00da8ff59ae037434fc" ] }, { "step": "Re-run", "detail": "Compare later sealed sweeps with the same method instead of turning this edition into a standing ranking.", "citation_ids": [ "evalevidence-0091d719054480ab6242" ] } ], "evidence": [ { "evidence_id": "evalevidence-bfa74cade8e7c59f1889", "input_id": "refusal-drift-current", "label": "Latest cross-lab family-level panel", "selector": "/models/*/{family_refusal_rate_pct,family_refusal_ci95_pct,n_families,controls_clean}", "value": { "openai/gpt-4o-mini": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": true }, "anthropic/claude-3-haiku": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": true }, "meta-llama/llama-3.3-70b-instruct": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": false }, "mistralai/mistral-nemo": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": true } }, "interpretation_limit": "The panel is a dated, non-representative set of named model endpoints.", "source_url": "https://palimpsest.info/readings/refusal-drift-latest.json" }, { "evidence_id": "evalevidence-e00da8ff59ae037434fc", "input_id": "refusal-drift-current", "label": "Family-level statistical method", "selector": "/{method,method_note,n_families,control_families}", "value": { "method": "every question is a family of meaning-preserving wordings; the family is the statistical unit; new refusals (answered->refused) are the erasure events; the standing alarm is a mixture supermartingale, so the lifetime false-alarm rate is bounded under unlimited peeking", "method_note": "Rates are family-level with Wilson 95% intervals. The paired test is an exact mid-p McNemar on a single transition and is not valid for the rolling series; the churn monitor is, by Ville's inequality. Control families are unremarkable questions: if they are refused, the run is an instrument fault and carries no censorship claim. The refusal classifier is lexical and is itself watched by a frozen anchor set.", "n_families": 41, "control_families": [ "civics/how-a-bill", "control/bread-baking", "control/kettle-limescale", "control/offside-rule", "control/photosynthesis", "control/ser-estar", "control/thunder-delay" ] }, "interpretation_limit": "Wilson intervals quantify sampling uncertainty inside the suite, not deployment-wide uncertainty.", "source_url": "https://palimpsest.info/readings/refusal-drift-latest.json" }, { "evidence_id": "evalevidence-0091d719054480ab6242", "input_id": "eval-registry", "label": "Sealed attestations for the latest panel", "selector": "/runs/@ts=2026-08-23T06:54:58.055463+00:00/@suite=frontier-overrefusal-v2", "value": { "anthropic/claude-3-haiku": { "seq": 522, "entry_hash": "69aa6d5ff4119d42054947a01b215d28d1c9c62e8fc68bd91e2c3f70493618ba", "responses_hash": "34f65e4a99f5e9806b2575b21d036fac419f4210e90b205673ac3f371e9f22f4" }, "meta-llama/llama-3.3-70b-instruct": { "seq": 524, "entry_hash": "e3f2d8ee0b3b355814171c4c43d56a6d9d7d9c7a90ac749d13323a0124dc9c7a", "responses_hash": "75135f5ffb2f9db91b75c1a9c1f25d0979bd99cf8313f0e675b8214534b412d2" }, "mistralai/mistral-nemo": { "seq": 526, "entry_hash": "61c775b7ac0a2177204144fecb7720c4e81b2094de55eed31a1f6dcd14a63839", "responses_hash": "d4c65d634dd655a8dca4f0b29d62bcdf7df525e27b32bea34cfd1600632a6638" }, "openai/gpt-4o-mini": { "seq": 528, "entry_hash": "507db5dbe1d7ad85733f2d1ccc9ef73191e871e403de9ce5c2bc7684d660a321", "responses_hash": "9703551a7f8d8323476b991e3f2b12ce1e6a5c15156a21994c41ca9dff4abbe5" } }, "interpretation_limit": "The chain proves these attestations persisted unchanged. It does not widen the sampled population.", "source_url": "https://palimpsest.info/readings/eval-registry.jsonl" } ], "evaluation_receipt": { "status": "passed", "publishable": true, "citation_coverage": 1.0, "sealed_run_count": 4, "gates": [ { "gate_id": "registry-chain", "label": "The eval registry verifies from genesis to the cited runs", "passed": true, "detail": "All 4 panel runs match verified registry attestations." }, { "gate_id": "controls-accounted-for", "label": "Control failures are visible and constrain the interpretation", "passed": true, "detail": "A failed control produces an instrument warning, never a censorship claim." }, { "gate_id": "uncertainty-visible", "label": "The article reports denominators and uncertainty with the rate", "passed": true, "detail": "Family counts and Wilson 95% interval bounds remain attached to the score." }, { "gate_id": "sentence-citations", "label": "Every analytical sentence names exact evidence receipts", "passed": true, "detail": "9 of 9 analytical sentences carry citations." }, { "gate_id": "adversarial-reading", "label": "Counterreadings, limitations, and reproduction steps are present", "passed": true, "detail": "The approved article shapes require all three surfaces before publication." }, { "gate_id": "bounded-authorship", "label": "No interviews or free-form model prose are represented as reporting", "passed": true, "detail": "Generated from sealed evaluation artifacts with a deterministic editorial template. No interviews and no free-form model prose were used." } ] }, "authorship": { "byline": "Palimpsest Eval Desk", "mode": "deterministic-eval-analysis", "human_interviews": "none", "freeform_model_generation": "none" }, "disclosure": "Generated from sealed evaluation artifacts with a deterministic editorial template. No interviews and no free-form model prose were used." }