"""Software checks on artificial numeric scoring-report fixtures only. No task answers, artifact ratings, study observations, or empirical estimates. """ import copy import json from pathlib import Path import random import subprocess import sys import tempfile import unittest import analyze_primary_contrast as analysis class PrimaryContrastTests(unittest.TestCase): def setUp(self): self.registry = analysis.read_json(analysis.BASE / "task-registry.json") self.treatment = "numeric-fixture-treatment" self.control = "numeric-fixture-control" self.repetitions = ["fixture-repeat-1", "fixture-repeat-2", "fixture-repeat-3"] self.task_ids = [t["task_id"] for t in self.registry["tasks"]] self.families = {f["family_id"]: f["task_ids"] for f in self.registry["families"]} def report(self, pattern=None): pattern = pattern or (lambda tid, rep, condition: 0) series = dict(artifact_scores=[], conditions={}) for cid in (self.treatment, self.control): task_means = {} for tid in self.task_ids: values = [] for index, rep in enumerate(self.repetitions): value = pattern(tid, index, cid) values.append(value) aid = "numeric-fixture-" + cid + tid + rep series["artifact_scores"].append(dict( blinded_artifact_id=aid, record_id="fixture-record-"+aid, condition_id=cid, task_id=tid, repetition_id=rep, score=value, eligible_for_aggregation=True, missing_criterion_ids=[])) task_means[tid] = sum(values)/len(values) family_scores = {f: sum(task_means[t] for t in tasks)/len(tasks) for f, tasks in self.families.items()} expected = len(self.task_ids)*len(self.repetitions) series["conditions"][cid] = dict( coverage_complete=True, expected_artifacts=expected, observed_records=expected, complete_records=expected, missing_assignments=[], missing_record_artifact_ids=[], incomplete_records=[], task_means=task_means, family_scores=family_scores, overall_score=sum(family_scores.values())/len(family_scores)) return dict(report_version="1.0", notice="Artificial numeric software fixture; not study data", coordinator_only=True, batch_id="numeric-fixture-batch", seal_record_id="fixture-not-a-real-seal", independent={"fixture-rater-a": copy.deepcopy(series), "fixture-rater-b": copy.deepcopy(series)}, adjudicated=series, raw_independent_agreement={}) def analyze(self, report, **kwargs): options = dict(series="adjudicated", replicates=200, seed=19) options.update(kwargs) return analysis.analyze_report(report, self.registry, self.treatment, self.control, **options) def test_whole_task_pairing_gives_zero_interval_for_identical_arms(self): # Task-level variability must not become unpaired arm variability. report = self.report(lambda tid, rep, cid: self.task_ids.index(tid)/12) result = self.analyze(report) self.assertEqual(result["mean_difference"], 0) self.assertEqual(result["confidence_interval_95"], [0, 0]) def test_all_repetitions_stay_in_each_task_cluster(self): # Individual paired repetitions differ, but every whole task difference # is zero. Resampling repetitions would introduce spurious uncertainty. report = self.report(lambda tid, rep, cid: ([0, 0.5, 1] if cid == self.treatment else [1, 0.5, 0])[rep]) result = self.analyze(report) self.assertEqual(result["confidence_interval_95"], [0, 0]) self.assertEqual(result["repetitions_per_task"], 3) for cluster in result["paired_task_clusters"]: self.assertEqual(cluster["repetition_ids"], self.repetitions) self.assertEqual(cluster["paired_task_difference"], 0) def test_task_repetition_and_family_weighting(self): first = self.task_ids[0] report = self.report(lambda tid, rep, cid: int(tid == first and rep == 0 and cid == self.treatment)) result = self.analyze(report) self.assertAlmostEqual(result["mean_difference"], 1/36) self.assertAlmostEqual(result["family_mean_differences"]["SYN"], 1/12) self.assertEqual(result["family_mean_differences"]["CON"], 0) def test_equal_family_engine_does_not_pool_unequal_stratum_sizes(self): # Mathematical unit check, distinct from the kit's fixed four-per-family registry. groups = {"fixture-A": [1, 1], "fixture-B": [0], "fixture-C": [0]} self.assertAlmostEqual(analysis.equal_family_mean(groups), 1/3) self.assertEqual(analysis.bootstrap_interval(groups, 20, 4), [1/3, 1/3]) def test_seed_reproducibility_and_input_order_invariance(self): report = self.report(lambda tid, rep, cid: self.task_ids.index(tid)/12 if cid == self.treatment else 0) before = copy.deepcopy(report) result = self.analyze(report, seed=82, replicates=400) self.assertEqual(result, self.analyze(report, seed=82, replicates=400)) self.assertEqual(report, before) random.Random(17).shuffle(report["adjudicated"]["artifact_scores"]) registry = copy.deepcopy(self.registry) registry["families"].reverse() reordered = analysis.analyze_report(report, registry, self.treatment, self.control, "adjudicated", replicates=400, seed=82) self.assertEqual(result, reordered) self.assertEqual(result["bootstrap"]["replicates"], 400) self.assertEqual(result["bootstrap"]["seed"], 82) def test_percentile_interpolation_is_explicit(self): self.assertAlmostEqual(analysis.percentile([0, 10, 20], 0.025), 0.5) self.assertAlmostEqual(analysis.percentile([0, 10, 20], 0.975), 19.5) def test_explicit_series_selection_never_pools_raters_or_adjudication(self): zero = self.report() one = self.report(lambda tid, rep, cid: int(cid == self.treatment)) one["independent"] = zero["independent"] self.assertEqual(self.analyze(one)["mean_difference"], 1) self.assertEqual(self.analyze(one, series="independent", rater_id="fixture-rater-a")["mean_difference"], 0) for kwargs in [dict(series="independent"), dict(series="adjudicated", rater_id="fixture-rater-a"), dict(series="pooled"), dict(series="independent", rater_id="absent")]: with self.subTest(kwargs=kwargs), self.assertRaises(analysis.ValidationError): self.analyze(one, **kwargs) def test_treatment_minus_control_sign(self): report = self.report(lambda tid, rep, cid: int(cid == self.treatment)) forward = self.analyze(report) reverse = analysis.analyze_report(report, self.registry, self.control, self.treatment, "adjudicated", replicates=20, seed=1) self.assertEqual(forward["mean_difference"], 1) self.assertEqual(reverse["mean_difference"], -1) def test_missing_row_and_falsely_complete_summary_rejected(self): report = self.report() report["adjudicated"]["artifact_scores"].pop() with self.assertRaises(analysis.ValidationError): self.analyze(report) def test_missing_null_ineligible_and_declared_coverage_holes_rejected(self): for field, value in [("score", None), ("score", True), ("score", float("inf")), ("eligible_for_aggregation", False), ("missing_criterion_ids", ["C1"])]: report = self.report() report["adjudicated"]["artifact_scores"][0][field] = value with self.subTest(field=field, value=value), self.assertRaises(analysis.ValidationError): self.analyze(report) for field, value in [("coverage_complete", False), ("expected_artifacts", 37), ("missing_assignments", [{"task_id": self.task_ids[0]}])]: report = self.report() report["adjudicated"]["conditions"][self.treatment][field] = value with self.subTest(field=field), self.assertRaises(analysis.ValidationError): self.analyze(report) def test_repetition_labels_must_pair_even_when_counts_and_scores_match(self): report = self.report() for row in report["adjudicated"]["artifact_scores"]: if row["condition_id"] == self.control: row["repetition_id"] = "unpaired-" + row["repetition_id"] with self.assertRaisesRegex(analysis.ValidationError, "paired coverage"): self.analyze(report) def test_duplicate_task_cell_and_record_rejected(self): report = self.report() duplicate = copy.deepcopy(report["adjudicated"]["artifact_scores"][0]) report["adjudicated"]["artifact_scores"].append(duplicate) with self.assertRaisesRegex(analysis.ValidationError, "duplicate"): self.analyze(report) duplicate.update(blinded_artifact_id="new-fixture-artifact", record_id="new-fixture-record") with self.assertRaisesRegex(analysis.ValidationError, "duplicate condition/task/repetition"): self.analyze(report) def test_incompatible_versions_and_summary_mismatch_rejected(self): report = self.report() report["report_version"] = "2.0" with self.assertRaisesRegex(analysis.ValidationError, "incompatible"): self.analyze(report) report = self.report() report["adjudicated"]["conditions"][self.treatment]["task_means"][self.task_ids[0]] = 0.5 with self.assertRaisesRegex(analysis.ValidationError, "task mean"): self.analyze(report) def test_invalid_bootstrap_configuration_and_condition_ids(self): for kwargs in [dict(replicates=1), dict(replicates=True), dict(seed=True), dict(seed=0.5)]: with self.subTest(kwargs=kwargs), self.assertRaises(analysis.ValidationError): self.analyze(self.report(), **kwargs) with self.assertRaises(analysis.ValidationError): analysis.analyze_report(self.report(), self.registry, self.treatment, self.treatment, "adjudicated") def test_cli_is_read_only_and_errors_produce_no_estimate(self): with tempfile.TemporaryDirectory(prefix="numeric-contrast-fixture-") as tmp: path = Path(tmp)/"artificial-report.json" path.write_text(json.dumps(self.report())) original = path.read_bytes() command = [sys.executable, "-B", str(analysis.BASE/"analyze_primary_contrast.py"), "--report", str(path), "--treatment-id", self.treatment, "--control-id", self.control, "--series", "adjudicated", "--replicates", "20", "--seed", "7"] run = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(run.returncode, 0, run.stderr) self.assertEqual(path.read_bytes(), original) self.assertEqual(json.loads(run.stdout)["source_scoring_report_version"], "1.0") invalid = self.report() invalid["report_version"] = "incompatible" path.write_text(json.dumps(invalid)) bad = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(bad.returncode, 2) self.assertEqual(bad.stdout, "") self.assertIn("error", json.loads(bad.stderr)) if __name__ == "__main__": unittest.main()