{ "schema_version": "palimpsest-eval-article.v1", "article_id": "evalarticle-635451d800aca8c43550", "revision_id": "evalarticlev-a24cbd73e705abef13639ae7", "previous_revision_id": "evalarticlev-46f2505bb21d25d21ce2722b", "slug": "zero-observed-is-not-zero-uncertainty", "url": "/journal/zero-observed-is-not-zero-uncertainty/", "kicker": "Eval method / uncertainty", "title": "Zero observed refusals is not zero uncertainty", "dek": "4 control-clean model runs observed 0 refused families out of 34, yet each zero result still carries a Wilson 95% upper bound of 10.2%. That interval is part of the finding, not fine print.", "thesis": "A finite eval can observe no refusals and still leave a meaningful range of plausible rates.", "finding_state": "bounded-finding", "published_at": "2026-08-14T13:34:43.929654+00:00", "updated_at": "2026-08-15T12:55:00.684023+00:00", "key_numbers": [ { "value": "0", "label": "refused families", "note": "in each of 4 clean runs", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0" ] }, { "value": "34", "label": "monitored non-control families", "note": "family is the statistical unit", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0", "evalevidence-e00da8ff59ae037434fc" ] }, { "value": "10.2%", "label": "95% upper interval bound", "note": "for a zero-of-34 result", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0", "evalevidence-e00da8ff59ae037434fc" ] } ], "sections": [ { "section_id": "what-zero-contains", "heading": "What zero contains", "paragraphs": [ { "sentences": [ { "text": "In the 15 Aug 2026 panel, 4 control-clean runs observed zero refused families among 34 monitored non-control families.", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0", "evalevidence-41bc5415f7f0e940c770" ] }, { "text": "For a zero-of-34 result, the published Wilson 95% interval still reaches 10.2%.", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0", "evalevidence-e00da8ff59ae037434fc" ] }, { "text": "Zero observed events and zero plausible event rate are different statements.", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0", "evalevidence-e00da8ff59ae037434fc" ] } ] } ] }, { "section_id": "unit-of-analysis", "heading": "The unit is a question family, not a prompt", "paragraphs": [ { "sentences": [ { "text": "Each monitored question can appear in several meaning-preserving wordings, but the family is the statistical unit.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] }, { "text": "That prevents a model from looking artificially precise merely because the same idea was phrased many times.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] } ] } ] }, { "section_id": "agreement-boundary", "heading": "Cross-lab agreement is useful and bounded", "paragraphs": [ { "sentences": [ { "text": "The latest panel contains 4 named endpoints, of which 4 passed every ordinary control.", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0" ] }, { "text": "Agreement across those endpoints is stronger than a one-model anecdote, but it is not a probability sample of all models, deployments, languages, or future releases.", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0", "evalevidence-e00da8ff59ae037434fc" ] } ] } ] }, { "section_id": "dated-not-ranked", "heading": "Read it as a dated panel, not a leaderboard", "paragraphs": [ { "sentences": [ { "text": "The registry binds every score to a model label, prompt commitment, response hash, and timestamp.", "citation_ids": [ "evalevidence-41bc5415f7f0e940c770" ] }, { "text": "Those receipts make change auditable, but they do not justify a permanent ranking from one sweep.", "citation_ids": [ "evalevidence-41bc5415f7f0e940c770", "evalevidence-e4d8af3496e0ac6acdd0" ] } ] } ] } ], "counterreadings": [ { "text": "Observing zero refused families in 4 clean runs is real favorable evidence within this suite.", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0" ] }, { "text": "The nonzero upper interval is not evidence that hidden refusals occurred. It is the uncertainty left by a finite sample.", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0", "evalevidence-e00da8ff59ae037434fc" ] } ], "limitations": [ { "text": "The monitored families were selected for an eval, not sampled from all questions people ask.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] }, { "text": "A provider may change routing or model behaviour after the dated sweep.", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0", "evalevidence-41bc5415f7f0e940c770" ] }, { "text": "A clean control state permits interpretation of the suite result but does not validate the lexical classifier against independent human coders.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] } ], "methodology": [ { "step": "Pre-register", "detail": "Commit the exact probe bank before any endpoint is queried.", "citation_ids": [ "evalevidence-41bc5415f7f0e940c770" ] }, { "step": "Group", "detail": "Treat a meaning-preserving prompt family as the statistical unit.", "citation_ids": [ "evalevidence-e00da8ff59ae037434fc" ] }, { "step": "Bound", "detail": "Report the denominator and Wilson 95% interval with every family rate.", "citation_ids": [ "evalevidence-e4d8af3496e0ac6acdd0", "evalevidence-e00da8ff59ae037434fc" ] }, { "step": "Re-run", "detail": "Compare later sealed sweeps with the same method instead of turning this edition into a standing ranking.", "citation_ids": [ "evalevidence-41bc5415f7f0e940c770" ] } ], "evidence": [ { "evidence_id": "evalevidence-e4d8af3496e0ac6acdd0", "input_id": "refusal-drift-current", "label": "Latest cross-lab family-level panel", "selector": "/models/*/{family_refusal_rate_pct,family_refusal_ci95_pct,n_families,controls_clean}", "value": { "openai/gpt-4o-mini": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": true }, "anthropic/claude-3-haiku": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": true }, "meta-llama/llama-3.3-70b-instruct": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": true }, "mistralai/mistral-nemo": { "family_refusal_rate_pct": 0.0, "family_refusal_ci95_pct": [ 0.0, 10.2 ], "n_refused_families": 0, "n_families": 34, "controls_clean": true } }, "interpretation_limit": "The panel is a dated, non-representative set of named model endpoints.", "source_url": "https://palimpsest.info/readings/refusal-drift-latest.json" }, { "evidence_id": "evalevidence-e00da8ff59ae037434fc", "input_id": "refusal-drift-current", "label": "Family-level statistical method", "selector": "/{method,method_note,n_families,control_families}", "value": { "method": "every question is a family of meaning-preserving wordings; the family is the statistical unit; new refusals (answered->refused) are the erasure events; the standing alarm is a mixture supermartingale, so the lifetime false-alarm rate is bounded under unlimited peeking", "method_note": "Rates are family-level with Wilson 95% intervals. The paired test is an exact mid-p McNemar on a single transition and is not valid for the rolling series; the churn monitor is, by Ville's inequality. Control families are unremarkable questions: if they are refused, the run is an instrument fault and carries no censorship claim. The refusal classifier is lexical and is itself watched by a frozen anchor set.", "n_families": 41, "control_families": [ "civics/how-a-bill", "control/bread-baking", "control/kettle-limescale", "control/offside-rule", "control/photosynthesis", "control/ser-estar", "control/thunder-delay" ] }, "interpretation_limit": "Wilson intervals quantify sampling uncertainty inside the suite, not deployment-wide uncertainty.", "source_url": "https://palimpsest.info/readings/refusal-drift-latest.json" }, { "evidence_id": "evalevidence-41bc5415f7f0e940c770", "input_id": "eval-registry", "label": "Sealed attestations for the latest panel", "selector": "/runs/@ts=2026-08-15T12:55:00.684023+00:00/@suite=frontier-overrefusal-v2", "value": { "openai/gpt-4o-mini": { "seq": 422, "entry_hash": "215340010da39cc08b8f38203607e3977cfb2c1298162295e0ed1738c5a0a6fd", "responses_hash": "1c1f62271dc91e3b41792e27b4aa228099282cd8ecd6fe40d0312d1595808a93" }, "anthropic/claude-3-haiku": { "seq": 424, "entry_hash": "2418486c95c09e2066ae102649c94fe71b939352022791b7f0b243b4f4224784", "responses_hash": "009f10fb045a3a4e4383115febbc9bda6d34c4979b7b927de2d72b5cb9fc7a5a" }, "meta-llama/llama-3.3-70b-instruct": { "seq": 426, "entry_hash": "952ecb8e2229206b4cad2cdf427243b9b87392478513cea2bcebef907c9ba7af", "responses_hash": "86991492d4f3515317af351e5002962a560b1710bf16dd621148215c5987967a" }, "mistralai/mistral-nemo": { "seq": 428, "entry_hash": "53228f91a9345813a7e3297a2b17dbfb020432522d072ecbd72bcd03a465aaf3", "responses_hash": "c1b3518fda0daa9f4b49cadc21c445bb0ef0fe24c82e4a660d72fbedd9a0b51b" } }, "interpretation_limit": "The chain proves these attestations persisted unchanged. It does not widen the sampled population.", "source_url": "https://palimpsest.info/readings/eval-registry.jsonl" } ], "evaluation_receipt": { "status": "passed", "publishable": true, "citation_coverage": 1.0, "sealed_run_count": 4, "gates": [ { "gate_id": "registry-chain", "label": "The eval registry verifies from genesis to the cited runs", "passed": true, "detail": "All 4 panel runs match verified registry attestations." }, { "gate_id": "controls-accounted-for", "label": "Control failures are visible and constrain the interpretation", "passed": true, "detail": "A failed control produces an instrument warning, never a censorship claim." }, { "gate_id": "uncertainty-visible", "label": "The article reports denominators and uncertainty with the rate", "passed": true, "detail": "Family counts and Wilson 95% interval bounds remain attached to the score." }, { "gate_id": "sentence-citations", "label": "Every analytical sentence names exact evidence receipts", "passed": true, "detail": "9 of 9 analytical sentences carry citations." }, { "gate_id": "adversarial-reading", "label": "Counterreadings, limitations, and reproduction steps are present", "passed": true, "detail": "The approved article shapes require all three surfaces before publication." }, { "gate_id": "bounded-authorship", "label": "No interviews or free-form model prose are represented as reporting", "passed": true, "detail": "Generated from sealed evaluation artifacts with a deterministic editorial template. No interviews and no free-form model prose were used." } ] }, "authorship": { "byline": "Palimpsest Eval Desk", "mode": "deterministic-eval-analysis", "human_interviews": "none", "freeform_model_generation": "none" }, "disclosure": "Generated from sealed evaluation artifacts with a deterministic editorial template. No interviews and no free-form model prose were used." }