{
  "checks": [
    {
      "applies_to": [
        "all-eval-suites"
      ],
      "dimension": "integrity",
      "evidence": "All 548 attestations verify with no chain or ordering break.",
      "id": "registry-chain-integrity",
      "label": "The public eval chain recomputes and every run follows a preregistration",
      "limitation": "Hash-chain integrity detects revision and ordering violations; it does not establish construct validity or external witnessing by itself.",
      "status": "pass",
      "verify_cmd": "python -m scripts.verify_eval_registry"
    },
    {
      "applies_to": [
        "frontier-overrefusal-v2"
      ],
      "dimension": "prompt_precommitment",
      "evidence": "144 published prompts reproduce the preregistered commitment.",
      "id": "frontier-exact-prompt-commitment",
      "label": "Exact frontier prompts were frozen before answers",
      "limitation": "Git history or an external witness is still needed to assign public wall-clock time to the commit.",
      "status": "pass",
      "verify_cmd": "python -m scripts.verify_refusal_transcripts"
    },
    {
      "applies_to": [
        "frontier-overrefusal-v2"
      ],
      "dimension": "response_recomputability",
      "evidence": "576 full responses reproduce all 4 current model seals.",
      "id": "frontier-response-recomputation",
      "label": "Raw frontier responses reproduce every current seal",
      "limitation": "Recomputation proves which text was measured, not that the project classifier is substantively correct.",
      "status": "pass",
      "verify_cmd": "python -m scripts.verify_refusal_transcripts"
    },
    {
      "applies_to": [
        "frontier-overrefusal-v2"
      ],
      "dimension": "pipeline_reproducibility",
      "evidence": "All 576 labels re-derive; the published artifact is method v4 and the shipping judge is v4.",
      "id": "frontier-label-recomputation",
      "label": "Published frontier labels re-derive from raw text",
      "limitation": "Agreement with the project's own deterministic judge is not human construct validation.",
      "status": "pass",
      "verify_cmd": "python -m scripts.verify_refusal_transcripts"
    },
    {
      "applies_to": [
        "frontier-overrefusal-v2"
      ],
      "dimension": "statistical_design",
      "evidence": "4 model readings carry family-level Wilson intervals, controls, power, and an anytime-valid panel monitor.",
      "id": "frontier-statistical-design",
      "label": "Frontier estimates disclose uncertainty, power, controls, and repeated-look correction",
      "limitation": "A quiet churn alarm cannot establish that a one-time permanent single-question erasure did not occur; the reading states this separately.",
      "status": "pass"
    },
    {
      "applies_to": [
        "cn-sensitive-generative-firewall-v2"
      ],
      "dimension": "prompt_precommitment",
      "evidence": "All 44 prompt arms reproduce a prior v2 registry commitment.",
      "id": "gfi-concept-id-commitment",
      "label": "GFI v2 freezes exact prompts, panel, cohorts, k, method, and classifier bytes",
      "limitation": "The public git commit carrying the protocol is the external before/after witness; the chain itself proves internal ordering.",
      "status": "pass",
      "verify_cmd": "python -m scripts.verify_gfi_transcripts"
    },
    {
      "applies_to": [
        "cn-sensitive-generative-firewall-v2"
      ],
      "dimension": "response_recomputability",
      "evidence": "All 3 preregistered panel model matrices reproduce their current registry seals.",
      "id": "gfi-response-recomputation",
      "label": "Every full GFI v2 sample matrix reproduces its model seal",
      "limitation": "Seal recomputation establishes the measured bytes, not human agreement with the classifier.",
      "status": "pass",
      "verify_cmd": "python -m scripts.verify_gfi_transcripts"
    },
    {
      "applies_to": [
        "cn-sensitive-generative-firewall-v2"
      ],
      "dimension": "statistical_design",
      "evidence": "The reading uses k=5 samples across 20 usable cells with a Wilson interval and explicit control/abstention gates.",
      "id": "gfi-statistical-design",
      "label": "GFI samples repeated cells and publishes uncertainty, abstention, and controls",
      "limitation": "The interval describes sampling variability under this panel and prompt bank; it is not a population estimate for all Chinese models or all censored topics.",
      "status": "pass"
    },
    {
      "applies_to": [
        "cn-sensitive-generative-firewall-v2"
      ],
      "dimension": "pipeline_reproducibility",
      "evidence": "The latest public GFI artifact is method v4; the shipping quote-aware judge is v4.",
      "id": "gfi-method-boundary",
      "label": "GFI method changes rebaseline the longitudinal series",
      "limitation": "Method identity alone does not validate the construct.",
      "status": "pass"
    },
    {
      "applies_to": [
        "all-lexically-classified-suites"
      ],
      "dimension": "construct_validation",
      "evidence": "The 145-row sample, codebook, thresholds, and falsifier are preregistered; no complete RESULT.json has been published.",
      "id": "independent-human-coding",
      "label": "Two-human blind classifier validation",
      "limitation": "Until two independent humans complete the study, refusal and party-line outputs remain labels from a published lexical rule—not validated human judgements.",
      "status": "pending",
      "verify_cmd": "python -m scripts.build_eval_assurance --check"
    },
    {
      "applies_to": [
        "all-eval-suites"
      ],
      "dimension": "independent_replication",
      "evidence": "The code, prompt banks, registry, transcripts, and verification commands are public and MIT licensed.",
      "id": "external-replication",
      "label": "Independent team replication",
      "limitation": "No unaffiliated team has yet published a preregistered replication against this registry.",
      "status": "open"
    }
  ],
  "claim_ceiling": {
    "can_claim": "Palimpsest publishes tamper-evident, statistically explicit eval outputs; both the frontier and China-focused suites bind exact protocols and let readers recompute current seals from full responses.",
    "cannot_yet_claim": "The lexical construct has not completed independent human validation, and no unaffiliated replication is on record.",
    "level": "provisional-measurement",
    "promotion_rule": "Do not promote the evals to human-validated until the preregistered two-coder study passes its frozen falsifiers; do not call them independently replicated until an unaffiliated preregistered run is sealed and published."
  },
  "dimensions": [
    {
      "checks": [
        "registry-chain-integrity"
      ],
      "id": "integrity",
      "question": "Are records internally tamper-evident and ordered?",
      "status": "pass"
    },
    {
      "checks": [
        "frontier-exact-prompt-commitment",
        "gfi-concept-id-commitment"
      ],
      "id": "prompt_precommitment",
      "question": "Were the questions fixed before answers?",
      "status": "pass"
    },
    {
      "checks": [
        "frontier-response-recomputation",
        "gfi-response-recomputation"
      ],
      "id": "response_recomputability",
      "question": "Can published raw evidence reproduce the seals?",
      "status": "pass"
    },
    {
      "checks": [
        "frontier-label-recomputation",
        "gfi-method-boundary"
      ],
      "id": "pipeline_reproducibility",
      "question": "Can labels be regenerated under an identified method?",
      "status": "pass"
    },
    {
      "checks": [
        "frontier-statistical-design",
        "gfi-statistical-design"
      ],
      "id": "statistical_design",
      "question": "Are denominators, uncertainty, controls, and repeated looks handled?",
      "status": "pass"
    },
    {
      "checks": [
        "independent-human-coding"
      ],
      "id": "construct_validation",
      "question": "Do independent humans support what the labels mean?",
      "status": "pending"
    },
    {
      "checks": [
        "external-replication"
      ],
      "id": "independent_replication",
      "question": "Has an unaffiliated team reproduced the finding?",
      "status": "open"
    }
  ],
  "generated_at": "2026-08-24T06:31:10.572234+00:00",
  "schema": "palimpsest.eval-assurance.v1",
  "sources": {
    "frontier_reading": "readings/refusal-drift-latest.json",
    "frontier_transcripts": "readings/refusal-drift-transcripts.json",
    "gfi_reading": "readings/latest.json",
    "human_validation": "validation/studies/2026-08-01-gfi-classifier-v1/",
    "registry": "readings/eval-registry.jsonl"
  },
  "summary": {
    "checks": 11,
    "fail": 0,
    "open": 1,
    "partial": 0,
    "pass": 9,
    "pending": 1
  },
  "title": "Palimpsest AI Eval Assurance",
  "verify": [
    "python -m scripts.verify_eval_registry",
    "python -m scripts.verify_refusal_transcripts",
    "python -m scripts.build_eval_assurance --check"
  ],
  "what": "A claim-by-claim audit of what the published eval evidence proves, partially supports, and does not yet establish."
}