{ "schema": "T_GPT_METRICS_V0_8_2", "version": "0.8.2", "date": "2026-09-11", "status": "STATISTICAL_REPAIR_BEFORE_CONFIRMATORY_FREEZE", "supersedes_primary_path": "metrics-v0.8.1.json", "preserves_history": true, "scope": "Observable semantic distribution shift, recovery-specific lift, aggregation coverage and mapping validity. No metric is interpreted as direct access to a latent total field or model weights.", "field_types": { "FIELD_TOK": "conditional token/continuation distribution when directly available", "FIELD_SEM": "empirical distribution over frozen semantic cluster identities plus explicit mapping-failure states", "FIELD_TRAJ": "paired or graph-structured generation/agent trajectories with provenance", "FIELD_VIS": "normalized propositions/configurations exposed by the evaluated aggregate" }, "primary_change_from_v0_8_1": "Hard support thresholds and set-difference ES_LOSS are removed from the confirmatory endpoint path. They remain descriptive/historical only.", "final_assignment_states": { "known_cluster": "cluster ID present in frozen codebook", "UNMAPPED": "semantic content not assignable to any frozen cluster after adjudication", "UNRESOLVED": "annotation/adjudication could not determine a stable assignment", "forbidden_in_final_metric_input": ["uncertain", "development_new_cluster_proposal", "development_split_proposal", "development_merge_proposal"] }, "probabilities": { "formula": "p_X(c) = n_X(c) / N_X_total_finalized", "denominator": "all finalized generation units in arm X, including UNMAPPED and UNRESOLVED states", "known_semantic_mass": "sum_c p_X(c) over frozen known clusters", "mapping_rates": ["UNMAPPED_rate", "UNRESOLVED_rate", "known_semantic_mass"], "rationale": "Mapping failure must not silently disappear from the denominator or alter a support threshold." }, "cluster_relevance": { "symbol": "R_t(c)", "unit": "frozen cluster, not individual output", "definition": "Independent blinded cluster-level relevance adjudication performed after development and before confirmatory analysis.", "allowed_values": [0, 1], "agreement_gate": "Krippendorff alpha nominal >= 0.67 required; >=0.80 target. If gate fails, return to development and issue a new freeze_id.", "author_primary_annotation_forbidden": true }, "continuous_shift": { "relevant_cluster_set": "C_t = {c in frozen codebook for task t : R_t(c)=1}", "DEFICIT_RAW": { "formula": "D_t(A,B) = sum_{c in C_t} max(0, p_A(c) - p_B(c))", "range": [0, 1], "interpretation": "Directional mass deficit of B relative to A across frozen relevant clusters." }, "EXPANSION_RAW": { "formula": "E_t(A,B) = sum_{c in C_t} max(0, p_B(c) - p_A(c))", "range": [0, 1], "interpretation": "Directional mass present more strongly in B than A. Contraction and expansion are always reported together." }, "non_smooth_warning": "Positive-part operators are directionally differentiable but not fully differentiable at equality points; standard within-task bootstrap validity is not assumed." }, "null_calibration": { "preferred_E2": "Independent same-condition reference replicate A_prime. Null floor N_t = 0.5 * [D_t(A,A_prime) + D_t(A_prime,A)].", "preferred_E1": "Label permutation/randomization under H0 for each adjacent stage contrast, preserving group sizes and frozen assignments.", "DEFICIT_EXCESS": "D_excess_t(A,B) = D_t(A,B) - null_floor_t", "EXPANSION_EXCESS": "E_excess_t(A,B) = E_t(A,B) - null_floor_t", "signed_values_allowed": true, "rule": "Do not truncate calibrated excess at zero in the primary analysis. Negative values are informative about sampling-floor overcorrection or reverse direction." }, "recovery": { "paired_unit": "Each contracted generation B_i is forked into one RECOVERY probe continuation and one NEUTRAL_CONTROL continuation from the same B_i state/context. Pair identity is preserved.", "cluster_deficit": "d_c = max(0, p_A(c) - p_B(c))", "closure": "C_t(A,B,R) = sum_{c in C_t} min(d_c, max(0, p_R(c) - p_B(c)))", "RECOVERY_LIFT": { "role": "PRIMARY_ENDPOINT_E2", "formula": "L_t = C_t(A,B,R_recovery) - C_t(A,B,R_control)", "range": [-1, 1], "defined_when_no_deficit": "0 by construction because both closure terms are 0", "interpretation": "Additional movement toward the reference semantic distribution attributable to a recovery-oriented probe beyond an equal-cost neutral recheck." }, "paired_randomization": "Within each B_i fork, swap RECOVERY vs CONTROL labels to construct the within-task null distribution for probe-specific lift.", "secondary_descriptive": { "RECOVERY_FRACTION_POOLED": "sum_t C_t(A,B,R_recovery) / sum_t D_t(A,B) when denominator > 0", "RECOVERY_LIFT_FRACTION_POOLED": "sum_t [C_recovery_t - C_control_t] / sum_t D_t(A,B) when denominator > 0", "rule": "Never average per-task recovery ratios." }, "retired_from_primary": ["REC_GAIN_v0.8.1", "IRREV_OBS"] }, "aggregation": { "AGG_COVERAGE": { "role": "PRIMARY_ENDPOINT_E3", "formula": "sum_{c in C_P} w_P(c) * represented_Y(c) / sum_{c in C_P} w_P(c)", "represented_Y": "Binary blinded adjudication that a normalized proposition in Y semantically entails or faithfully expresses frozen cluster c under the representation codebook.", "weights": "w_P(c)=p_P(c) by default; sensitivity uses uniform weights" }, "REPRESENTATION_FPR": { "role": "mandatory guardrail", "formula": "# distractor clusters judged represented / # preregistered distractor clusters", "distractors": "Plausible frozen cluster descriptions absent from the actual pool, mixed blindly with true pool clusters." }, "REPRESENTATION_PRECISION": { "role": "secondary", "formula": "TP / (TP + FP) over evaluated true-pool clusters and distractors when denominator > 0" }, "output_budget": "All aggregators receive identical maximum output-token budget and the same frozen branch pool.", "normalization": "Before representation annotation, aggregate outputs are converted to a fixed proposition-list format under a condition-blind normalization procedure; raw outputs are preserved for audit." }, "minority_analysis": { "status": "secondary_only", "definition": "Absolute preregistered frequency band in the frozen pool, not a data-defined quartile.", "default_band": [0.02, 0.08], "minimum_cluster_count": 8, "rule": "If fewer than 8 relevant clusters fall in the band, no minority-recall estimate is reported for that task/family." }, "descriptive_metrics": { "D_sem_thresholded": "Historical/descriptive only; never primary.", "H_sem": "Descriptive entropy across known relevant clusters; plugin estimator bias is acknowledged and no confirmatory claim depends on H_sem.", "mapping_rates": "Always reported by condition/stage." }, "inference": { "within_task": "Use randomization/permutation procedures matched to the design; do not use the v0.8.1 percentile bootstrap on thresholded or positive-part plug-in statistics.", "across_tasks": "Task is the upper-level inferential unit. Generalized confirmatory claims use task-level estimates and task-level resampling/modeling only.", "task_bootstrap": "Allowed for the distribution of task-level primary effects once the preregistered minimum task gate is satisfied; this is not a bootstrap claim about the within-task positive-part functional.", "multiple_comparisons": "Exactly one primary endpoint and one primary contrast per experiment. Secondary contrasts use Holm correction within family or are explicitly exploratory.", "effect_reporting": "Report task-level distribution, mean/median effect, uncertainty, mapping rates, quality guardrails and cost." }, "primary_endpoint_by_experiment": { "E1_STAGE": "one preregistered stage contrast on DEFICIT_EXCESS; transition decomposition is secondary", "E2_RECOVERY": "mean task-level RECOVERY_LIFT", "E3_TOPOLOGY": "mean task-level AGG_COVERAGE with mandatory REPRESENTATION_FPR guardrail", "E4_R_META": "difference in blinded structural-recurrence rate between STRUCTURAL_CRITIQUE and pooled non-structural controls" }, "mapping_validity": { "no_silent_exclusion": true, "primary_interpretation_rule": "If mapping/adjudication quality is inadequate, label the cell MEASUREMENT_INSUFFICIENT rather than converting mapping failure into semantic contraction.", "required_reports": ["cluster_assignment_alpha", "cluster_relevance_alpha", "UNMAPPED_rate", "UNRESOLVED_rate", "blindness_guess_accuracy"] } }