"""Artificial numeric fixtures only: no task answers or study observations. Fixtures reuse task IDs/schema as metadata. Temporary IDs, timestamps, hashes, evidence strings, and binary patterns are software-test inputs, not run records. """ import copy import hashlib import json from pathlib import Path import subprocess import sys import tempfile import unittest import score_forms as scoring class ScoringTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix="numeric-scoring-fixture-") self.addCleanup(self.temp.cleanup) self.directory = Path(self.temp.name) self.registry = scoring.read_json(scoring.BASE / "task-registry.json") self.rating_template = scoring.read_json(scoring.BASE / "rating-form.json") self.adjudication_template = scoring.read_json(scoring.BASE / "adjudication-form.json") self.raters = ["fixture-rater-a", "fixture-rater-b"] self.manifest = dict(manifest_version="1.0", batch_id="numeric-fixture-batch", seal_record_id="numeric-fixture-seal-not-a-run", independent_rater_ids=self.raters, conditions=[dict(condition_id="numeric-fixture-condition", repetition_ids=["fixture-repeat-1"])], assignments=[]) self.forms = [] for rater in self.raters: form = copy.deepcopy(self.rating_template) form.update(form_id=rater+"-form", batch_id=self.manifest["batch_id"], seal_record_id=self.manifest["seal_record_id"], rater_id=rater, locked_at_utc="2000-01-01T00:01:00Z", ratings=[]) self.forms.append(form) self.add_repetition("fixture-repeat-1") def add_repetition(self, rep): if rep not in self.manifest["conditions"][0]["repetition_ids"]: self.manifest["conditions"][0]["repetition_ids"].append(rep) for template in self.rating_template["ratings"]: aid = "numeric-fixture-" + template["task_id"] + "-" + rep digest = hashlib.sha256(aid.encode()).hexdigest() self.manifest["assignments"].append(dict( blinded_artifact_id=aid, blinded_artifact_sha256=digest, task_id=template["task_id"], task_version=template["task_version"], rubric_version=template["rubric_version"], family_id=template["family_id"], condition_id="numeric-fixture-condition", repetition_id=rep)) for rater, form in zip(self.raters, self.forms): row = copy.deepcopy(template) row.update(rating_record_id=rater+"-"+aid, blinded_artifact_id=aid, blinded_artifact_sha256=digest, assessment_status="complete", completed_at_utc="2000-01-01T00:00:00Z") for c in row["criteria"]: c.update(satisfaction=1, artifact_evidence_locations=["Artificial numeric fixture locator"], public_basis_locations=["Fixture mapping only; no task answer"], reason="Numeric test input only") form["ratings"].append(row) def set_values(self, row, values): for c, value in zip(row["criteria"], values): c["satisfaction"] = value row["assessment_status"] = "needs_resolution" if None in values else "complete" def adjudication(self): form = copy.deepcopy(self.adjudication_template) form.update(form_id="numeric-fixture-adjudication", batch_id=self.manifest["batch_id"], seal_record_id=self.manifest["seal_record_id"], adjudicator_id="fixture-third-person", source_rating_form_ids=[f["form_id"] for f in self.forms], independent_rater_ids=self.raters, locked_at_utc="2000-01-01T00:03:00Z", adjudications=[]) for left, right in zip(self.forms[0]["ratings"], self.forms[1]["ratings"]): template = next(r for r in self.adjudication_template["adjudications"] if r["task_id"] == left["task_id"]) row = copy.deepcopy(template) row.update(adjudication_record_id="fixture-decision-"+left["blinded_artifact_id"], source_rating_record_ids=[left["rating_record_id"], right["rating_record_id"]], adjudication_status="complete", completed_at_utc="2000-01-01T00:02:00Z") for key in scoring.IDENTITY: row[key] = left[key] for c, lc, rc in zip(row["criteria"], left["criteria"], right["criteria"]): pair = [lc["satisfaction"], rc["satisfaction"]] disagreement = None if None in pair else pair[0] != pair[1] value = next((v for v in reversed(pair) if v is not None), 0) route = ("missing_or_ambiguous_rating_resolved" if disagreement is None else "adjudicator_decision" if disagreement else "agreement_carried_forward") c.update(independent_satisfaction=pair, disagreement=disagreement, resolution=route, adjudicated_satisfaction=value, artifact_evidence_locations=["Artificial numeric fixture locator"], public_basis_locations=["Fixture mapping only"], resolution_rationale="Numeric test input only") form["adjudications"].append(row) return form def write(self, name, value): path = self.directory / name path.write_text(json.dumps(value), encoding="utf-8") return path def run_score(self, adjudication=None): manifest = self.write("coordinator-fixture.json", self.manifest) paths = [self.write(f"rating-{i}.json", f) for i, f in enumerate(self.forms)] adj_paths = [] if adjudication is None else [self.write("adjudication.json", adjudication)] return scoring.score_assessment(scoring.BASE / "task-registry.json", manifest, paths, adj_paths) def condition(self, report, rater=0): return report["independent"][self.raters[rater]]["conditions"]["numeric-fixture-condition"] def test_complete_scores_and_input_files_unchanged(self): self.run_score(self.adjudication()) before = {p: p.read_bytes() for p in self.directory.iterdir()} result = scoring.score_assessment(scoring.BASE / "task-registry.json", self.directory/"coordinator-fixture.json", [self.directory/"rating-0.json", self.directory/"rating-1.json"], [self.directory/"adjudication.json"]) self.assertEqual(self.condition(result)["overall_score"], 1) self.assertEqual(result["raw_independent_agreement"]["overall"]["compared_pairs"], 144) self.assertTrue(all(p.read_bytes() == content for p, content in before.items())) def test_missing_value_is_not_zero_or_smaller_denominator(self): self.set_values(self.forms[0]["ratings"][0], [None] + [1]*11) result = self.run_score() # Sorted IDs need not have the first task first; find the missing row. missing = [r for r in result["independent"][self.raters[0]]["artifact_scores"] if r["missing_criterion_ids"]] self.assertIsNone(missing[0]["score"]) self.assertFalse(self.condition(result)["coverage_complete"]) self.assertTrue(all(v is None for v in self.condition(result)["family_scores"].values())) self.assertEqual(self.condition(result, 1)["overall_score"], 1) agreement = result["raw_independent_agreement"]["overall"] self.assertEqual((agreement["matching_pairs"], agreement["compared_pairs"], agreement["missing_pairs"]), (143, 143, 1)) def test_invalid_binaries(self): for invalid in [True, False, 0.0, 1.0, "1", -1, 2, 0.5]: with self.subTest(value=invalid): self.forms[0]["ratings"][0]["criteria"][0]["satisfaction"] = invalid with self.assertRaises(scoring.ValidationError): self.run_score() def test_mismatched_artifact_hash(self): self.forms[1]["ratings"][0]["blinded_artifact_sha256"] = "0"*64 with self.assertRaisesRegex(scoring.ValidationError, "identity"): self.run_score() def test_mismatched_adjudication_identity(self): adj = self.adjudication() adj["adjudications"][0]["task_id"] = "DEV-SYN-02" with self.assertRaisesRegex(scoring.ValidationError, "identity"): self.run_score(adj) def test_task_version_and_mapping_mismatch(self): self.forms[0]["ratings"][0]["rubric_version"] = "changed" with self.assertRaises(scoring.ValidationError): self.run_score() self.forms[0]["ratings"][0]["rubric_version"] = "1.0" self.forms[0]["ratings"][0]["criteria"][0]["public_requirement_id"] = "R2" with self.assertRaisesRegex(scoring.ValidationError, "criterion/requirement"): self.run_score() def test_duplicate_record_and_rater_artifact(self): duplicate = copy.deepcopy(self.forms[0]["ratings"][0]) self.forms[0]["ratings"].append(duplicate) with self.assertRaisesRegex(scoring.ValidationError, "duplicate record"): self.run_score() duplicate["rating_record_id"] = "different-ID-same-rating" with self.assertRaisesRegex(scoring.ValidationError, "duplicate rater/artifact"): self.run_score() def test_duplicate_criterion(self): self.forms[0]["ratings"][0]["criteria"][1] = copy.deepcopy(self.forms[0]["ratings"][0]["criteria"][0]) with self.assertRaisesRegex(scoring.ValidationError, "duplicate criterion"): self.run_score() def test_uneven_record_coverage_blocks_all_family_scores(self): self.add_repetition("fixture-repeat-2") self.forms[0]["ratings"].pop() result = self.run_score() condition = self.condition(result) self.assertEqual((condition["expected_artifacts"], condition["complete_records"]), (24, 23)) self.assertIsNone(condition["overall_score"]) self.assertTrue(all(v is None for v in condition["family_scores"].values())) def test_missing_manifest_cell_is_reported_not_silently_dropped(self): self.add_repetition("fixture-repeat-2") self.manifest["assignments"].pop() for form in self.forms: form["ratings"].pop() condition = self.condition(self.run_score()) self.assertEqual(len(condition["missing_assignments"]), 1) self.assertEqual(condition["expected_artifacts"], 24) self.assertIsNone(condition["overall_score"]) def test_equal_repetition_task_and_family_weighting(self): self.add_repetition("fixture-repeat-2") for form in self.forms: for row in form["ratings"]: self.set_values(row, [0]*12) self.set_values(form["ratings"][12], [1]*12) condition = self.condition(self.run_score()) self.assertEqual(condition["task_means"]["DEV-SYN-01"], 0.5) self.assertEqual(condition["family_scores"]["SYN"], 0.125) self.assertAlmostEqual(condition["overall_score"], 1/24) def test_independent_and_adjudicated_series_remain_distinct(self): for row in self.forms[0]["ratings"]: self.set_values(row, [0]*12) for row in self.forms[1]["ratings"]: self.set_values(row, [1]*6 + [0]*6) result = self.run_score(self.adjudication()) self.assertEqual(self.condition(result)["overall_score"], 0) self.assertEqual(self.condition(result, 1)["overall_score"], 0.5) self.assertEqual(result["adjudicated"]["conditions"]["numeric-fixture-condition"]["overall_score"], 0.5) self.assertEqual(result["raw_independent_agreement"]["overall"]["raw_agreement"], 0.5) def test_adjudication_does_not_fill_original_missing_rating(self): self.set_values(self.forms[0]["ratings"][0], [None]+[1]*11) result = self.run_score(self.adjudication()) self.assertIsNone(self.condition(result)["overall_score"]) self.assertEqual(result["adjudicated"]["conditions"]["numeric-fixture-condition"]["overall_score"], 1) self.assertIsNone(self.forms[0]["ratings"][0]["criteria"][0]["satisfaction"]) def test_copied_judgments_and_resolution_routes_validated(self): adj = self.adjudication() adj["adjudications"][0]["criteria"][0]["independent_satisfaction"] = [0, 1] with self.assertRaisesRegex(scoring.ValidationError, "copied independent"): self.run_score(adj) adj = self.adjudication() adj["adjudications"][0]["criteria"][0]["adjudicated_satisfaction"] = 0 with self.assertRaisesRegex(scoring.ValidationError, "carried forward"): self.run_score(adj) def test_stale_stored_score_rejected(self): self.forms[0]["ratings"][0]["scores"]["requirement_satisfaction_score"] = 0 with self.assertRaisesRegex(scoring.ValidationError, "stored score"): self.run_score() def test_method_metadata_not_allowed_in_rater_form(self): self.forms[0]["condition_id"] = "leaked-condition" with self.assertRaisesRegex(scoring.ValidationError, "method metadata"): self.run_score() def test_manifest_duplicate_cell_and_undeclared_repetition(self): self.manifest["assignments"].append(copy.deepcopy(self.manifest["assignments"][0])) with self.assertRaisesRegex(scoring.ValidationError, "duplicate manifest"): self.run_score() self.manifest["assignments"].pop() self.manifest["assignments"][0]["repetition_id"] = "undeclared" with self.assertRaisesRegex(scoring.ValidationError, "undeclared"): self.run_score() def test_duplicate_json_keys_rejected(self): path = self.directory/"duplicate-key.json" path.write_text('{"same": 1, "same": 0}') with self.assertRaisesRegex(scoring.ValidationError, "duplicate JSON key"): scoring.read_json(path) def test_malformed_empty_containers_are_not_missing_coverage(self): self.forms[0]["ratings"] = {} with self.assertRaisesRegex(scoring.ValidationError, "ratings: expected array"): self.run_score() self.forms[0]["ratings"] = [] self.manifest["assignments"] = {} with self.assertRaisesRegex(scoring.ValidationError, "assignments: expected array"): self.run_score() def test_duplicate_form_and_adjudication_records(self): self.forms[1]["form_id"] = self.forms[0]["form_id"] with self.assertRaisesRegex(scoring.ValidationError, "duplicate form"): self.run_score() self.forms[1]["form_id"] = self.raters[1]+"-form" adj = self.adjudication() duplicate = copy.deepcopy(adj["adjudications"][0]) duplicate["adjudication_record_id"] = "other-adjudication-ID" adj["adjudications"].append(duplicate) with self.assertRaisesRegex(scoring.ValidationError, "duplicate adjudicated artifact"): self.run_score(adj) def test_incomplete_status_and_incomplete_adjudication_hold_summaries(self): self.forms[0]["ratings"][0]["assessment_status"] = "needs_resolution" result = self.run_score() self.assertIsNone(self.condition(result)["overall_score"]) adj = self.adjudication() row = adj["adjudications"][0] row["adjudication_status"] = "needs_resolution" row["criteria"][0].update(adjudicated_satisfaction=None, resolution="unresolved") result = self.run_score(adj) self.assertIsNone(result["adjudicated"]["conditions"]["numeric-fixture-condition"]["overall_score"]) def test_zero_agreement_denominator_and_cli(self): for form in self.forms: for row in form["ratings"]: self.set_values(row, [None]*12) result = self.run_score() self.assertEqual(result["raw_independent_agreement"]["overall"]["compared_pairs"], 0) self.assertIsNone(result["raw_independent_agreement"]["overall"]["raw_agreement"]) command = [sys.executable, "-B", str(scoring.BASE/"score_forms.py"), "--manifest", str(self.directory/"coordinator-fixture.json"), "--ratings", str(self.directory/"rating-0.json"), str(self.directory/"rating-1.json")] run = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(run.returncode, 0, run.stderr) self.assertIsNone(json.loads(run.stdout)["independent"][self.raters[0]]["conditions"]["numeric-fixture-condition"]["overall_score"]) self.forms[0]["ratings"][0]["criteria"][0]["satisfaction"] = True self.write("rating-0.json", self.forms[0]) bad = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(bad.returncode, 2) self.assertEqual(bad.stdout, "") self.assertIn("error", json.loads(bad.stderr)) if __name__ == "__main__": unittest.main()