"""Artificial numeric fixtures only, not task answers or empirical findings.""" import copy from datetime import datetime, timedelta, timezone import hashlib import json from pathlib import Path import subprocess import sys import tempfile import unittest import analyze_requirement_regression as regression import test_score_forms as scoring_fixtures class RegressionTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix="artificial-requirement-regression-") self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) self.manifest_path = self.root/"coordinator-regression.json" self.fixtures = {} for phase in ("earlier", "final"): fixture = scoring_fixtures.ScoringTests() fixture.setUp() self.addCleanup(fixture.doCleanups) old_batch, old_seal = fixture.manifest["batch_id"], fixture.manifest["seal_record_id"] fixture.manifest["batch_id"] = phase+"-"+old_batch fixture.manifest["seal_record_id"] = phase+"-"+old_seal for assignment in fixture.manifest["assignments"]: assignment["blinded_artifact_id"] = phase+"-"+assignment["blinded_artifact_id"] for form in fixture.forms: form["form_id"] = phase+"-"+form["form_id"] form["batch_id"] = fixture.manifest["batch_id"] form["seal_record_id"] = fixture.manifest["seal_record_id"] for row in form["ratings"]: row["rating_record_id"] = phase+"-"+row["rating_record_id"] row["blinded_artifact_id"] = phase+"-"+row["blinded_artifact_id"] self.fixtures[phase] = fixture task = self.fixtures["earlier"].manifest["assignments"][0] self.manifest = dict(manifest_version="1.0", analysis_id="artificial-numeric-regression", run_batch_seal=dict(seal_record_id="fixture-run-seal-not-an-actual-run", sealed=True, sealed_at_utc="1999-12-31T00:00:00Z"), independent_rater_ids=["analysis-rater-a", "analysis-rater-b"], earlier_assessments={}, final_assessments={}, pairs=[dict( pair_id="fixture-pair", run_id="fixture-run", condition_id=task["condition_id"], task_id=task["task_id"], task_version=task["task_version"], rubric_version=task["rubric_version"], repetition_id=task["repetition_id"], earlier=self.endpoint("earlier", 2), final=self.endpoint("final", 10))]) def endpoint(self, phase, turn, index=0): row = self.fixtures[phase].manifest["assignments"][index] return {**{k: row[k] for k in ("blinded_artifact_id", "blinded_artifact_sha256")}, "turn": turn} def values(self, phase, rater, values): fixture = self.fixtures[phase] fixture.set_values(fixture.forms[rater]["ratings"][0], values) def descriptor(self, path): return dict(path=str(path), sha256=hashlib.sha256(path.read_bytes()).hexdigest()) def prepare(self, adjudicated=True): for phase, fixture in self.fixtures.items(): coordinator = fixture.write("coordinator.json", fixture.manifest) ratings = [fixture.write(f"source-{i}.json", form) for i, form in enumerate(fixture.forms)] adj_paths = [fixture.write("decisions.json", fixture.adjudication())] if adjudicated else [] self.manifest[phase+"_assessments"] = dict( coordinator_manifest=self.descriptor(coordinator), rating_forms=[self.descriptor(p) for p in ratings], adjudication_forms=[self.descriptor(p) for p in adj_paths], rater_mapping=dict(zip(self.manifest["independent_rater_ids"], fixture.raters))) self.manifest_path.write_text(json.dumps(self.manifest)) def analyze(self, adjudicated=True): self.prepare(adjudicated) return regression.analyze_regression(self.manifest_path) def test_binary_transitions_and_previously_satisfied_denominator(self): self.values("earlier", 0, [1, 1, 0, 0]+[1]*8) self.values("final", 0, [0, 1, 1, 0]+[1]*8) report = self.analyze() series = report["independent"]["analysis-rater-a"] self.assertEqual([r["transition"] for r in series["pairs"][0]["criteria"][:4]], ["1->0", "1->1", "0->1", "0->0"]) self.assertEqual(series["totals"]["eligible_previously_satisfied_denominator"], 10) self.assertEqual(series["totals"]["observed_regressions"], 1) self.assertEqual(series["totals"]["assessed_regression_rate"], 0.1) def test_missing_judgments_are_not_passes_or_failures(self): self.values("earlier", 0, [1, 1, 1, 0, None]+[0]*7) self.values("final", 0, [0, 1, None, 1, 0]+[0]*7) series = self.analyze()["independent"]["analysis-rater-a"] summary = series["totals"] self.assertEqual(summary["eligible_previously_satisfied_denominator"], 2) self.assertEqual(summary["observed_regressions"], 1) self.assertEqual(summary["assessed_regression_rate"], 0.5) self.assertEqual(summary["missing_criterion_pairs"], 2) self.assertEqual(summary["known_earlier_satisfied"], 3) self.assertEqual(summary["missing_final_for_known_satisfied"], 1) self.assertIsNone(series["pairs"][0]["criteria"][2]["regressed"]) self.assertIsNone(series["pairs"][0]["criteria"][4]["transition"]) def test_no_eligible_denominator_is_null_not_zero(self): self.values("earlier", 0, [0]*12) self.values("final", 0, [1]*12) totals = self.analyze()["independent"]["analysis-rater-a"]["totals"] self.assertEqual(totals["eligible_previously_satisfied_denominator"], 0) self.assertIsNone(totals["assessed_regression_rate"]) self.assertEqual(totals["missing_criterion_pairs"], 0) def test_independent_raters_and_adjudication_remain_separate(self): self.values("earlier", 0, [1]*12) self.values("final", 0, [0]*12) self.values("earlier", 1, [0]*12) self.values("final", 1, [1]*12) report = self.analyze() self.assertEqual(report["independent"]["analysis-rater-a"]["totals"]["assessed_regression_rate"], 1) self.assertIsNone(report["independent"]["analysis-rater-b"]["totals"]["assessed_regression_rate"]) self.assertIsNone(report["adjudicated"]["totals"]["assessed_regression_rate"]) def test_adjudication_does_not_fill_missing_independent_values(self): self.values("earlier", 0, [None]+[1]*11) self.values("final", 0, [0]*12) self.values("final", 1, [0]*12) report = self.analyze() self.assertEqual(report["independent"]["analysis-rater-a"]["totals"]["missing_criterion_pairs"], 1) self.assertEqual(report["adjudicated"]["totals"]["eligible_previously_satisfied_denominator"], 12) self.assertIsNone(report["independent"]["analysis-rater-a"]["pairs"][0]["criteria"][0]["earlier_satisfaction"]) def test_missing_records_have_explicit_pair_coverage(self): self.fixtures["earlier"].forms[0]["ratings"].pop(0) series = self.analyze(adjudicated=False)["independent"]["analysis-rater-a"] self.assertEqual(series["totals"]["missing_endpoint_record_pairs"], 1) self.assertEqual(series["totals"]["missing_criterion_pairs"], 12) self.assertIsNone(series["totals"]["assessed_regression_rate"]) def test_criteria_pair_by_ids_not_row_order(self): self.values("earlier", 0, [1]+[0]*11) self.values("final", 0, [0, 1]+[0]*10) self.fixtures["final"].forms[0]["ratings"][0]["criteria"].reverse() result = self.analyze(adjudicated=False)["independent"]["analysis-rater-a"]["pairs"][0] self.assertEqual(result["criteria"][0]["criterion_id"], "C1") self.assertTrue(result["criteria"][0]["regressed"]) self.assertEqual(result["summary"]["observed_regressions"], 1) def test_rater_alias_mapping_is_explicit_and_not_pooling(self): fixture = self.fixtures["final"] fixture.raters = ["final-source-alias-a", "final-source-alias-b"] fixture.manifest["independent_rater_ids"] = fixture.raters for form, rater in zip(fixture.forms, fixture.raters): form["rater_id"] = rater report = self.analyze() self.assertEqual(set(report["independent"]), {"analysis-rater-a", "analysis-rater-b"}) self.manifest["final_assessments"]["rater_mapping"]["analysis-rater-a"] = fixture.raters[1] self.manifest_path.write_text(json.dumps(self.manifest)) with self.assertRaisesRegex(regression.scoring.ValidationError, "must not pool"): regression.analyze_regression(self.manifest_path) def test_duplicate_pairs_cells_and_reversed_endpoints_rejected(self): original = copy.deepcopy(self.manifest["pairs"]) self.manifest["pairs"].append(copy.deepcopy(original[0])) with self.assertRaisesRegex(regression.scoring.ValidationError, "duplicate pair"): self.analyze() self.manifest["pairs"][1].update(pair_id="other-pair", run_id="other-run-same-cell") with self.assertRaisesRegex(regression.scoring.ValidationError, "duplicate pair"): self.analyze() self.manifest["pairs"] = original self.manifest["pairs"][0]["earlier"]["turn"] = 10 with self.assertRaisesRegex(regression.scoring.ValidationError, "reversed or equal"): self.analyze() def test_task_rubric_and_artifact_hash_mismatches_rejected(self): original = copy.deepcopy(self.manifest["pairs"][0]) self.manifest["pairs"][0]["final"] = self.endpoint("final", 10, index=1) with self.assertRaisesRegex(regression.scoring.ValidationError, "task/rubric/run-cell/hash mismatch"): self.analyze() self.manifest["pairs"][0] = copy.deepcopy(original) self.manifest["pairs"][0]["rubric_version"] = "2.0" with self.assertRaisesRegex(regression.scoring.ValidationError, "incompatible"): self.analyze() self.manifest["pairs"][0] = copy.deepcopy(original) self.manifest["pairs"][0]["final"]["blinded_artifact_sha256"] = "0"*64 with self.assertRaisesRegex(regression.scoring.ValidationError, "hash mismatch"): self.analyze() def test_shared_validator_rejects_invalid_binary_and_mapping(self): self.fixtures["final"].forms[0]["ratings"][0]["criteria"][0]["satisfaction"] = True with self.assertRaises(regression.scoring.ValidationError): self.analyze(adjudicated=False) self.fixtures["final"].forms[0]["ratings"][0]["criteria"][0]["satisfaction"] = 1 self.fixtures["final"].forms[0]["ratings"][0]["criteria"][0]["public_requirement_id"] = "R2" with self.assertRaisesRegex(regression.scoring.ValidationError, "criterion/requirement mismatch"): self.analyze(adjudicated=False) def test_unsealed_future_and_preseal_assessments_rejected(self): self.manifest["run_batch_seal"]["sealed"] = False with self.assertRaisesRegex(regression.scoring.ValidationError, "declared sealed"): self.analyze() self.manifest["run_batch_seal"]["sealed"] = True self.manifest["run_batch_seal"]["sealed_at_utc"] = (datetime.now(timezone.utc)+timedelta(days=1)).isoformat() with self.assertRaisesRegex(regression.scoring.ValidationError, "future"): self.analyze() self.manifest["run_batch_seal"]["sealed_at_utc"] = "2001-01-01T00:00:00Z" with self.assertRaisesRegex(regression.scoring.ValidationError, "predates"): self.analyze() def test_no_best_history_or_inference_of_absent_assessments(self): self.values("earlier", 0, [0]*12) report = self.analyze(adjudicated=False) self.assertEqual(len(report["independent"]["analysis-rater-a"]["pairs"]), 1) self.assertEqual(report["independent"]["analysis-rater-a"]["totals"]["known_earlier_satisfied"], 0) self.assertEqual(report["adjudicated"]["totals"]["missing_criterion_pairs"], 12) self.assertIsNone(report["adjudicated"]["totals"]["assessed_regression_rate"]) def test_source_hashes_read_only_and_cli_error_behavior(self): self.prepare() paths = [Path(d["path"]) for phase in ("earlier_assessments", "final_assessments") for d in [self.manifest[phase]["coordinator_manifest"], *self.manifest[phase]["rating_forms"], *self.manifest[phase]["adjudication_forms"]]] before = {p: p.read_bytes() for p in paths+[self.manifest_path]} command = [sys.executable, "-B", str(regression.BASE/"analyze_requirement_regression.py"), "--manifest", str(self.manifest_path)] run = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(run.returncode, 0, run.stderr) self.assertEqual(json.loads(run.stdout)["regression_report_version"], "1.0") self.assertTrue(all(p.read_bytes() == data for p, data in before.items())) self.manifest["earlier_assessments"]["rating_forms"][0]["sha256"] = "0"*64 self.manifest_path.write_text(json.dumps(self.manifest)) bad = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(bad.returncode, 2) self.assertEqual(bad.stdout, "") self.assertIn("hash mismatch", json.loads(bad.stderr)["error"]) if __name__ == "__main__": unittest.main()