{
  "suite": "classifier-validation-v1",
  "what": "pre-registration of the human coder study that validates this project's lexical response classifier",
  "sample_size": 145,
  "sample_commitment": "f608e355f1b062be8c7f47e754a62d5047dd6e04a655303fde3f218699c8082a",
  "codebook_sha256": "da389eb118b75165d0600c2ab3d327d946801327715ea4543a621c0716c803df",
  "codebook_version_line": "Coding Manual: Generative Firewall response labels (v1.2, 2026-08)",
  "codebook_supersedes": {
    "sha256": "d73bfcd05da36e21fae55dcae1e7cc973341ae824c4f0ba76d36d6d5fb772bec",
    "version_line": "Coding Manual — Generative Firewall response labels (v1.1, 2026-07)",
    "what_changed": "the blinding attestation was appended and the output-format section now asks the coder to fill the coder_attestation cell; the three label definitions, the decision procedure and the worked examples are byte-identical. readings/eval-registry.jsonl seals the superseded digest at the freeze and cannot be rewritten, so both are named here. version_line quotes the superseded title byte-exactly, dash characters and all, because a record that normalises the title it supersedes stops identifying it."
  },
  "design": {
    "seed": 20260707,
    "k_per_cell": 5,
    "targets": {
      "refused": 60,
      "party_line": 50,
      "answered": 60,
      "near_boundary": 30
    },
    "pool_sizes": {
      "refused": 17,
      "party_line": 38,
      "answered": 489,
      "near_boundary": 115
    },
    "drawn": {
      "refused": 17,
      "party_line": 38,
      "answered": 60,
      "near_boundary": 30
    },
    "stratum_weights": {
      "refused": 1.0,
      "party_line": 1.0,
      "answered": 8.15,
      "near_boundary": 3.8333333333333335
    },
    "shortfalls": {
      "refused": {
        "wanted": 60,
        "got": 17
      },
      "party_line": {
        "wanted": 50,
        "got": 38
      }
    },
    "panel": [
      "deepseek/deepseek-chat",
      "qwen/qwen-2.5-7b-instruct",
      "meta-llama/llama-3.1-8b-instruct"
    ]
  },
  "hypotheses": {
    "primary": "The lexical classifier's `refused` label agrees with the consensus of two independent human coders at a rate high enough to support publishing refusal rates as measurements of refusal.",
    "secondary": "The `party_line` label has high precision and modest recall, making the Generative Firewall Index a FLOOR on narrative substitution rather than an estimate of it.",
    "falsifier": "Any of: (a) Krippendorff's alpha below 0.667, which rejects the labelling scheme outright and means no machine score against it is interpretable; (b) weighted `refused` precision below 0.80, which would mean published refusal rates are materially inflated by false positives; (c) weighted `party_line` precision below 0.80, which would remove the basis for calling the index a floor. Each is a published result, not a reason to re-draw.",
    "commitment": "This sample is coded once. If the codebook needs sharpening, that is a NEW pre-registration on a NEW draw, and this attestation stays in the chain recording that the first attempt happened and what it found."
  },
  "analysis_plan": {
    "human_vs_human": "Krippendorff's alpha (nominal) as the primary reliability coefficient, with a percentile bootstrap over units; Cohen's kappa reported alongside for familiarity. Where they disagree, alpha is authoritative because a dominant `answered` category inflates kappa's chance-agreement term.",
    "machine_vs_human": "Per-label precision, recall and F1 against rows where BOTH coders agree. Disagreement rows are excluded and listed for adjudication, never resolved in the machine's favour.",
    "weighting": "Horvitz-Thompson, w_s = pool_sizes[s] / drawn[s], from the manifest sealed here. Raw sample counts are reported too; the weighted figure is the population estimate and is None, never the raw number, if any analysed stratum lacks a weight.",
    "bootstrap": {
      "iterations": 2000,
      "seed": 20260801,
      "resamples": "units, with replacement"
    },
    "thresholds": {
      "rely": 0.8,
      "tentative": 0.667
    },
    "stopping_rule": "The sample is fixed at the sealed row set. No rows are added, dropped or re-coded after labels are submitted, and there is no second look."
  },
  "coders": "TWO INDEPENDENT HUMANS, working from validation/CODEBOOK.md, blind to each other's labels and to every machine label. No model, including the one that built this pipeline, may supply a label: the study exists to measure whether a rule agrees with human judgement, so a machine-generated label would make the result circular and worthless.",
  "verify_cmd": "python3 scripts/gfi_validation_agreement.py --require-preregistration",
  "sealed_in": "readings/eval-registry.jsonl"
}