"""Artificial text/reference fixtures only; no study packets or ratings. All generated bundles live in TemporaryDirectory and are removed after tests. The registry/form metadata shapes are reused; fixture references contain no answers. """ import copy from datetime import datetime, timedelta, timezone import hashlib import json from pathlib import Path import subprocess import sys import tempfile import unittest import build_assessment_packets as builder def sha(content): return hashlib.sha256(content).hexdigest() class PacketBuilderTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix="artificial-packet-fixture-") self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) self.registry = builder.read_json(builder.BASE/"task-registry.json") self.references = self.root/"fixture-references" self.references.mkdir() for task in self.registry["tasks"]: for field, directory in (("public_packet", "public"), ("assessor_guide", "guides")): relative = directory+"/"+task["task_id"]+".txt" content = ("Artificial "+field+" fixture for "+task["task_id"]+"; no task answers.\r\n").encode() path = self.references/relative path.parent.mkdir(exist_ok=True) path.write_bytes(content) task[field] = dict(path=relative, sha256=sha(content)) conventions = b"Artificial common conventions fixture; no study observations.\n" (self.references/"conventions.md").write_bytes(conventions) self.registry["common_conventions"] = dict(path="conventions.md", declared_version=None, sha256=sha(conventions)) self.registry_path = self.references/"task-registry.json" self.registry_path.write_text(json.dumps(self.registry)) self.manifest = dict(build_manifest_version="1.0", batch_id="PRIVATE-BATCH-SENTINEL", seal_record_id="PRIVATE-SEAL-SENTINEL", batch_sealed=True, sealed_at_utc="2000-01-01T00:02:00Z", independent_rater_ids=["PRIVATE-RATER-ONE", "PRIVATE-RATER-TWO"], conditions=[dict(condition_id="PRIVATE-CONDITION-RED", repetition_ids=["PRIVATE-REPEAT-ONE"])], runs=[]) self.source_bytes = {} for index, task in enumerate(self.registry["tasks"][:6]): # Byte-identical outputs on distinct source paths are legitimate; # identity is not inferred from textual difference. content = "Artificial text fixture.\r\nUnicode café.\r\n".encode("utf-8") filename = f"PRIVATE-CONDITION-RED-source-{index}.txt" (self.root/filename).write_bytes(content) self.source_bytes[filename] = content self.manifest["runs"].append(dict( run_id=f"PRIVATE-RUN-{index}", condition_id="PRIVATE-CONDITION-RED", task_id=task["task_id"], task_version=task["task_version"], rubric_version=task["rubric_version"], repetition_id="PRIVATE-REPEAT-ONE", sealed=True, sealed_at_utc="2000-01-01T00:01:00Z", final_artifact=dict(artifact_id=f"PRIVATE-ARTIFACT-{index}", path=filename, sha256=sha(content)))) self.manifest_path = self.root/"coordinator-input.json" self.packets = self.root/"rater-packets" self.coordinator = self.root/"coordinator-only" def build(self, packets=None, coordinator=None, seed=0): self.manifest_path.write_text(json.dumps(self.manifest)) return builder.build_packets(self.manifest_path, packets or self.packets, coordinator or self.coordinator, self.registry_path, seed) def assert_no_outputs(self): self.assertFalse(self.packets.exists()) self.assertFalse(self.coordinator.exists()) def test_bytes_identities_references_and_blank_forms_preserved(self): result = self.build() self.assertEqual(result["artifact_count"], 6) key = builder.read_json(self.coordinator/"packet-key.json") scoring = builder.read_json(self.coordinator/"coordinator-manifest.json") tasks, _, _ = builder.registry_data(self.registry, self.registry_path.parent) builder.manifest_data(scoring, tasks) # Existing scorer contract; no ratings calculated. by_id = {a["blinded_artifact_id"]: a for a in key["artifacts"]} seen_forms, seen_records = set(), set() for alias in scoring["independent_rater_ids"]: root = self.packets/alias index = builder.read_json(root/"packet-index.json") form = builder.read_json(root/"rating-form.json") self.assertEqual(form["batch_id"], scoring["batch_id"]) self.assertEqual(form["seal_record_id"], scoring["seal_record_id"]) self.assertEqual(form["rater_id"], alias) self.assertIsNone(form["locked_at_utc"]) self.assertNotIn(form["form_id"], seen_forms) seen_forms.add(form["form_id"]) for item, row in zip(index["artifacts"], form["ratings"]): aid = item["blinded_artifact_id"] self.assertRegex(aid, r"^artifact-[0-9a-f]{32}$") self.assertEqual(row["blinded_artifact_id"], aid) self.assertEqual(row["blinded_artifact_sha256"], by_id[aid]["blinded_artifact_sha256"]) self.assertEqual((root/item["artifact_path"]).read_bytes(), Path(by_id[aid]["source_path"]).read_bytes()) task = tasks[item["task_id"]] for field, index_field in (("public_packet", "public_packet_path"), ("assessor_guide", "assessor_guide_path")): self.assertEqual(sha((root/item[index_field]).read_bytes()), task[field]["sha256"]) self.assertEqual(sha((root/item["conventions_path"]).read_bytes()), self.registry["common_conventions"]["sha256"]) self.assertNotIn(row["rating_record_id"], seen_records) seen_records.add(row["rating_record_id"]) self.assertTrue(all(c["satisfaction"] is None and c["reason"] is None and c["artifact_evidence_locations"] == [] for c in row["criteria"])) self.assertEqual(row["scores"], dict(criterion_count=12, satisfied_criteria=None, requirement_satisfaction_score=None)) for filename, content in self.source_bytes.items(): self.assertEqual((self.root/filename).read_bytes(), content) self.assertTrue((self.packets/"BUILD_COMPLETE").is_file()) self.assertTrue((self.coordinator/"BUILD_COMPLETE").is_file()) def test_metadata_is_outside_rater_files_and_names(self): self.build() for path in self.packets.rglob("*"): self.assertNotIn("PRIVATE-", str(path.relative_to(self.packets))) if path.is_file(): self.assertNotIn(b"PRIVATE-", path.read_bytes()) key_bytes = (self.coordinator/"packet-key.json").read_bytes() for sensitive in (b"PRIVATE-CONDITION-RED", b"PRIVATE-RUN-0", b"PRIVATE-REPEAT-ONE", b"PRIVATE-BATCH-SENTINEL"): self.assertIn(sensitive, key_bytes) for path in self.packets.rglob("rating-form.json"): value = builder.read_json(path) self.assertNotIn("condition_id", value) self.assertTrue(all("repetition_id" not in r and "run_id" not in r for r in value["ratings"])) def test_intrinsic_disclosure_is_not_redacted(self): source = self.manifest["runs"][0]["final_artifact"] content = b"Artificial fixture explicitly says PRIVATE-CONDITION-RED.\r\n" (self.root/source["path"]).write_bytes(content) source["sha256"] = sha(content) self.build() key = builder.read_json(self.coordinator/"packet-key.json") aid = next(a["blinded_artifact_id"] for a in key["artifacts"] if a["source_artifact_id"] == source["artifact_id"]) for alias in key["rater_aliases"].values(): self.assertEqual((self.packets/alias/"artifacts"/(aid+".txt")).read_bytes(), content) def test_hash_mismatch_prevents_all_outputs(self): source = self.manifest["runs"][0]["final_artifact"] (self.root/source["path"]).write_bytes(b"Changed artificial fixture") with self.assertRaisesRegex(builder.ValidationError, "hash mismatch"): self.build() self.assert_no_outputs() def test_reference_and_conventions_hash_mismatches(self): guide = self.registry["tasks"][0]["assessor_guide"] (self.references/guide["path"]).write_bytes(b"Changed artificial guide") with self.assertRaisesRegex(builder.ValidationError, "hash mismatch"): self.build() self.assert_no_outputs() # Restore guide, then corrupt the independent shared conventions source. task = self.registry["tasks"][0] content = ("Artificial assessor_guide fixture for "+task["task_id"]+"; no task answers.\r\n").encode() (self.references/guide["path"]).write_bytes(content) (self.references/"conventions.md").write_bytes(b"Changed conventions") with self.assertRaisesRegex(builder.ValidationError, "hash mismatch"): self.build() self.assert_no_outputs() def test_unsealed_batch_and_run_are_rejected(self): for value in (False, None, "true"): self.manifest["batch_sealed"] = value with self.subTest(value=value), self.assertRaises(builder.ValidationError): self.build() self.assert_no_outputs() self.manifest["batch_sealed"] = True self.manifest["runs"][0]["sealed"] = False with self.assertRaisesRegex(builder.ValidationError, "run must be sealed"): self.build() self.assert_no_outputs() def test_versions_and_seal_chronology_rejected(self): row = self.manifest["runs"][0] row["rubric_version"] = "not-1.0" with self.assertRaisesRegex(builder.ValidationError, "version mismatch"): self.build() row["rubric_version"] = "1.0" row["sealed_at_utc"] = "2000-01-01T00:03:00Z" with self.assertRaisesRegex(builder.ValidationError, "after declared batch"): self.build() self.assert_no_outputs() def test_future_seal_cannot_declare_already_sealed_batch(self): self.manifest["sealed_at_utc"] = (datetime.now(timezone.utc)+timedelta(days=1)).isoformat() with self.assertRaisesRegex(builder.ValidationError, "future"): self.build() self.assert_no_outputs() def test_duplicate_source_run_artifact_and_cell_identities(self): original = copy.deepcopy(self.manifest) self.manifest["runs"][1]["run_id"] = self.manifest["runs"][0]["run_id"] with self.assertRaisesRegex(builder.ValidationError, "duplicate run"): self.build() self.manifest = copy.deepcopy(original) self.manifest["runs"][1]["final_artifact"]["artifact_id"] = self.manifest["runs"][0]["final_artifact"]["artifact_id"] with self.assertRaisesRegex(builder.ValidationError, "duplicate source artifact"): self.build() self.manifest = copy.deepcopy(original) self.manifest["runs"][1]["task_id"] = self.manifest["runs"][0]["task_id"] with self.assertRaisesRegex(builder.ValidationError, "duplicate condition/task/repetition"): self.build() self.assert_no_outputs() def test_existing_outputs_are_never_overwritten(self): self.coordinator.mkdir() sentinel = self.coordinator/"unchanged.txt" sentinel.write_bytes(b"Existing coordinator contents") with self.assertRaisesRegex(builder.ValidationError, "refusing existing output"): self.build() self.assertEqual(sentinel.read_bytes(), b"Existing coordinator contents") self.assertFalse(self.packets.exists()) # A separate existing rater root is also rejected, even if empty. self.packets.mkdir() with self.assertRaisesRegex(builder.ValidationError, "refusing existing output"): self.build() def test_nested_and_symlink_outputs_rejected(self): with self.assertRaises(builder.ValidationError): self.build(coordinator=self.packets/"private-key") with self.assertRaises(builder.ValidationError): self.build(packets=self.coordinator/"raters") self.packets.symlink_to(self.root/"absent-target") with self.assertRaisesRegex(builder.ValidationError, "existing output"): self.build() self.assertTrue(self.packets.is_symlink()) def test_same_seed_reproduces_source_order_with_fresh_opaque_ids(self): self.build(seed=31) second_packets = self.root/"second-raters" second_key = self.root/"second-key" self.build(second_packets, second_key, seed=31) def source_orders(directory): key = builder.read_json(directory/"packet-key.json") sources = {a["blinded_artifact_id"]: a["source_artifact_id"] for a in key["artifacts"]} return {raw: [sources[x] for x in key["presentations"][alias]["artifact_order"]] for raw, alias in key["rater_aliases"].items()}, key first_orders, first = source_orders(self.coordinator) second_orders, second = source_orders(second_key) self.assertEqual(first_orders, second_orders) self.assertNotEqual(first["batch_id"], second["batch_id"]) self.assertEqual(first["shuffle_seed"], 31) def test_extra_prompt_metadata_and_non_utf8_artifacts_rejected(self): self.manifest["prompts"] = ["must not distribute"] with self.assertRaises(builder.ValidationError): self.build() self.manifest.pop("prompts") source = self.manifest["runs"][0]["final_artifact"] (self.root/source["path"]).write_bytes(b"\xff\xfe") source["sha256"] = sha(b"\xff\xfe") with self.assertRaisesRegex(builder.ValidationError, "UTF-8"): self.build() self.assert_no_outputs() def test_cli_uses_fixture_sources_and_never_generates_ratings(self): self.manifest_path.write_text(json.dumps(self.manifest)) before = self.manifest_path.read_bytes() command = [sys.executable, "-B", str(builder.BASE/"build_assessment_packets.py"), "--manifest", str(self.manifest_path), "--registry", str(self.registry_path), "--packets-dir", str(self.packets), "--coordinator-dir", str(self.coordinator)] run = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(run.returncode, 0, run.stderr) self.assertEqual(self.manifest_path.read_bytes(), before) self.assertEqual(json.loads(run.stdout)["rater_bundle_count"], 2) repeated = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(repeated.returncode, 2) self.assertEqual(repeated.stdout, "") self.assertIn("refusing existing output", json.loads(repeated.stderr)["error"]) if __name__ == "__main__": unittest.main()