#!/usr/bin/env python3 """Offline paired primary-contrast analysis of a version 1.0 scoring report. Development analysis tooling only; no rating recalculation or model execution. """ import argparse import hashlib import math from pathlib import Path import random import sys import json from score_forms import ValidationError, read_json, require, text BASE = Path(__file__).resolve().parent NOTICE = "Development analysis tooling, not experimental evidence; inference depends on the supplied data and design." def score(value, label): require(type(value) in (int, float) and math.isfinite(value) and 0 <= value <= 1, f"{label}: expected a finite score in [0,1], not null or boolean") return value def mean(values): return math.fsum(values) / len(values) def equal_family_mean(groups): """Average task differences within families, then give each family equal weight.""" return mean([mean(groups[f]) for f in sorted(groups)]) def percentile(sorted_values, probability): """Linear interpolation at zero-based rank probability * (B - 1).""" rank = probability * (len(sorted_values) - 1) lo = math.floor(rank) hi = math.ceil(rank) return sorted_values[lo] + (rank - lo) * (sorted_values[hi] - sorted_values[lo]) def bootstrap_interval(groups, replicates, seed): require(type(replicates) is int and replicates >= 2, "replicates must be an integer >= 2") require(type(seed) is int, "seed must be an integer") rng = random.Random(seed) draws = [] # Each element is a whole paired task's repetition-mean difference. Sampling # it once carries all of that task's repetitions and both comparator arms. for _ in range(replicates): family_means = [] for family in sorted(groups): tasks = groups[family] family_means.append(mean([tasks[rng.randrange(len(tasks))] for _ in tasks])) draws.append(mean(family_means)) draws.sort() return [percentile(draws, 0.025), percentile(draws, 0.975)] def registry_families(registry): require(registry["kit_version"] == "1.0", "unsupported task-registry version") tasks = registry["tasks"] require(isinstance(tasks, list), "registry tasks must be an array") by_id = {} for task in tasks: tid = text(task["task_id"], "registry task_id") require(tid not in by_id, "duplicate registry task") by_id[tid] = task require(isinstance(registry["families"], list), "registry families must be an array") families = {} for family in registry["families"]: fid = text(family["family_id"], "registry family_id") group = family["task_ids"] require(isinstance(group, list) and len(group) == 4, "expected four tasks per family") require(fid not in families, "duplicate registry family") for tid in group: require(tid in by_id and by_id[tid]["family_id"] == fid, "inconsistent registry family membership") families[fid] = sorted(group) members = [tid for group in families.values() for tid in group] require(len(families) == 3 and len(members) == len(set(members)) == len(by_id) == 12, "this development kit requires twelve distinct tasks in three families") return families def analyze_report(report, registry, treatment_id, control_id, series, rater_id=None, replicates=10000, seed=0): """Validate and analyze already-scored artifacts; never read criterion ratings.""" require(report["report_version"] == "1.0", "incompatible scoring report_version; expected 1.0") require(report["coordinator_only"] is True, "expected coordinator-only scoring report") text(report["batch_id"], "batch_id") text(report["seal_record_id"], "seal_record_id") text(treatment_id, "treatment_id") text(control_id, "control_id") require(treatment_id != control_id, "treatment and control must differ") if series == "independent": text(rater_id, "rater_id (required for independent series)") require(rater_id in report["independent"], "selected independent rater absent from report") selected = report["independent"][rater_id] else: require(series == "adjudicated", "select exactly independent or adjudicated") require(rater_id is None, "rater_id is not permitted for adjudicated series") selected = report["adjudicated"] families = registry_families(registry) task_ids = {t for group in families.values() for t in group} conditions = selected["conditions"] require(isinstance(conditions, dict) and treatment_id in conditions and control_id in conditions, "both selected condition IDs must exist in the selected series") rows = selected["artifact_scores"] require(isinstance(rows, list), "artifact_scores must be an array") cells = {c: {t: {} for t in task_ids} for c in (treatment_id, control_id)} seen_artifacts, seen_records = set(), set() for row in rows: cid = text(row["condition_id"], "artifact condition_id") if cid not in cells: continue # Other conditions/series are not pooled into this contrast. tid = row["task_id"] require(tid in task_ids, "selected contrast has an unknown task") rep = text(row["repetition_id"], "repetition_id") aid = text(row["blinded_artifact_id"], "blinded_artifact_id") rid = text(row["record_id"], "record_id") require(aid not in seen_artifacts and rid not in seen_records, "duplicate artifact or record ID in selected contrast") seen_artifacts.add(aid) seen_records.add(rid) require(rep not in cells[cid][tid], "duplicate condition/task/repetition cell") require(row["eligible_for_aggregation"] is True and row["missing_criterion_ids"] == [], "incomplete or ineligible selected artifact score") cells[cid][tid][rep] = score(row["score"], "artifact score") common_repetitions = None for cid in (treatment_id, control_id): condition = conditions[cid] require(condition["coverage_complete"] is True, "selected condition has incomplete declared coverage") for field in ("missing_assignments", "missing_record_artifact_ids", "incomplete_records"): require(type(condition[field]) is list and condition[field] == [], "selected condition has coverage holes") require(isinstance(condition["task_means"], dict) and set(condition["task_means"]) == task_ids, "missing/extra task means in selected condition") require(isinstance(condition["family_scores"], dict) and set(condition["family_scores"]) == set(families), "missing/extra family scores in selected condition") for tid in sorted(task_ids): repetitions = set(cells[cid][tid]) require(repetitions, "selected task has no scored repetitions") if common_repetitions is None: common_repetitions = repetitions require(repetitions == common_repetitions, "incomplete paired coverage: repetition IDs must match across both arms and every task") supplied = score(condition["task_means"][tid], "reported task mean") actual = mean([cells[cid][tid][r] for r in sorted(repetitions)]) require(math.isclose(supplied, actual, rel_tol=0, abs_tol=1e-12), "reported task mean disagrees with scored repetitions") expected = len(task_ids) * len(common_repetitions) for field in ("expected_artifacts", "observed_records", "complete_records"): require(type(condition[field]) is int and condition[field] == expected, "selected condition coverage count disagrees with paired rows") family_values = [] for family in sorted(families): actual = mean([condition["task_means"][t] for t in families[family]]) supplied = score(condition["family_scores"][family], "reported family score") require(math.isclose(actual, supplied, rel_tol=0, abs_tol=1e-12), "inconsistent reported family score") family_values.append(supplied) require(math.isclose(score(condition["overall_score"], "reported overall score"), mean(family_values), rel_tol=0, abs_tol=1e-12), "inconsistent reported overall score") groups, clusters = {}, [] for family in sorted(families): groups[family] = [] for tid in families[family]: # Reuse reported task means, after cross-checking them against the # existing artifact scores. No criterion/rater judgment is rescored. treatment_mean = conditions[treatment_id]["task_means"][tid] control_mean = conditions[control_id]["task_means"][tid] difference = treatment_mean - control_mean groups[family].append(difference) clusters.append(dict(task_id=tid, family_id=family, repetition_ids=sorted(common_repetitions), treatment_mean=treatment_mean, control_mean=control_mean, paired_task_difference=difference)) interval = bootstrap_interval(groups, replicates, seed) return dict(analysis_report_version="1.0", source_scoring_report_version="1.0", notice=NOTICE, coordinator_only=True, development_defaults_unfrozen=True, batch_id=report["batch_id"], seal_record_id=report["seal_record_id"], treatment_id=treatment_id, control_id=control_id, selected_series=dict(kind=series, rater_id=rater_id), contrast="treatment repetition-mean score minus control repetition-mean score, equally weighted by family", paired_task_count=len(task_ids), repetitions_per_task=len(common_repetitions), paired_task_clusters=clusters, family_mean_differences={f: mean(groups[f]) for f in sorted(groups)}, mean_difference=equal_family_mean(groups), confidence_interval_95=interval, bootstrap=dict(method="seeded percentile, task-clustered and family-stratified", replicates=replicates, seed=seed, confidence_level=0.95, percentile_interpolation="linear at q*(B-1)", task_sampling="with replacement, original task count within each family", repetitions="all repetitions and both conditions retained together per sampled task")) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--report", type=Path, required=True) parser.add_argument("--registry", type=Path, default=BASE / "task-registry.json") parser.add_argument("--treatment-id", required=True) parser.add_argument("--control-id", required=True) parser.add_argument("--series", choices=("independent", "adjudicated"), required=True) parser.add_argument("--rater-id") parser.add_argument("--replicates", type=int, default=10000) parser.add_argument("--seed", type=int, default=0) args = parser.parse_args(argv) try: result = analyze_report(read_json(args.report), read_json(args.registry), args.treatment_id, args.control_id, args.series, args.rater_id, args.replicates, args.seed) result["source_report_sha256"] = hashlib.sha256(args.report.read_bytes()).hexdigest() result["task_registry_sha256"] = hashlib.sha256(args.registry.read_bytes()).hexdigest() except (ValidationError, OSError, KeyError, TypeError, ValueError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())