"""Temporary artificial public-text, metadata and binary fixtures; no study findings.""" import contextlib import copy from datetime import datetime, timezone import io import json from pathlib import Path import sys import unittest from unittest import mock import assess_critical as critical sys.path.insert(0, str(Path(__file__).resolve().parent.parent/"run-sealing")) import test_seal_runs as fixtures class CriticalTests(unittest.TestCase): def setUp(self): self.f = fixtures.SealingTests() self.f.setUp() self.addCleanup(self.f.doCleanups) f = self.f self.public = {} for task in f.registry["tasks"]: tid = task["task_id"] text = "\ufeff# "+tid+": Artificial public fixture\r\n\r\n## Public requirements\r\n\r\n" text += "\r\n".join(f"- **R{i}:** Artificial public constraint {i}; no extra symbolic gate." for i in range(1, 13)) text += "\r\n\r\n## Complete authorized source packet\r\n\r\n### S1 — Artificial source A\r\n\r\nPUBLIC-A: synthetic α data.\r\n\r\n### S2 — Artificial source B\r\n\r\nPUBLIC-B: synthetic β data.\r\n" self.public[tid] = text content = text.encode("utf-8") (f.refs/task["public_packet"]["path"]).write_bytes(content) task["public_packet"]["sha256"] = critical.sealing.sha(content) f.registry_path.write_bytes(critical.sealing.json_bytes(f.registry)) f.source["records"] = [r for r in f.source["records"] if r["record_id"] != "fixture-usage"] for run in f.source["runs"]: run["record_ids"].remove("fixture-usage") f.source["conditions"].append(dict(condition_id="fixture-second-condition", repetition_ids=["fixture-repetition"])) f.source["runs"][1]["condition_id"] = "fixture-second-condition" f.set_availability() for run in f.source["runs"]: run["record_availability"]["usage_records"] = dict(state="unavailable", note="Artificial usage unavailable") f.seal() commit = json.loads((f.output/"packet-build-manifest.json").read_bytes()) self.manifest = dict(critical_manifest_version="1.0", assessment_id="artificial-critical-assessment", independent_rater_ids=["source-rater-a", "source-rater-b"], adjudicator_id="source-adjudicator", archives=[dict(archive_id="fixture-archive", path=f.output.name)], artifacts=[], designations=[]) self.designations, self.designation_paths = {}, {} for index, run in enumerate(commit["runs"]): row = {k: run[k] for k in ("run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id")} row.update(archive_id="fixture-archive", seal_record_id=commit["seal_record_id"], artifact_id=run["final_artifact"]["artifact_id"], artifact_sha256=run["final_artifact"]["sha256"]) self.manifest["artifacts"].append(row) tid = row["task_id"] packet = self.public[tid] requirements = [] if index == 0: for rid in ("R1", "R2"): start = packet.index("- **"+rid+":**") end = packet.index("\r\n", start) quote = "PUBLIC-A: synthetic α data." loc = packet.index(quote) requirements.append(dict(requirement_id=rid, requirement_location=dict(start=start, end=end, text=packet[start:end]), source_locations=[dict(source_id="S1", start=loc, end=loc+len(quote), text=quote)])) self.designations[tid] = dict(designation_version="1.0", task_id=tid, task_version="1.0", rubric_version="1.0", source_packet_sha256=critical.sealing.sha(packet.encode("utf-8")), review_status="reviewed", designator_id="private-human-designator", completed_at_utc="2000-01-01T00:00:00Z", note="Artificial metadata only: "+row["condition_id"], requirements=requirements) self.designation_paths[tid] = f.root/(tid+"-designation.json") self.manifest["designations"].append(dict(task_id=tid, designation=None, unavailability_reason=None)) self.manifest_path = f.root/"critical-selection.json" self.raters, self.coordinator = f.root/"critical-raters", f.root/"private-critical-key" self.key_path = self.coordinator/"critical-key.json" self.save() def now(self): return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") @property def tid(self): return self.manifest["artifacts"][0]["task_id"] def save(self): for row in self.manifest["designations"]: tid = row["task_id"] data = critical.sealing.json_bytes(self.designations[tid]) self.designation_paths[tid].write_bytes(data) row["designation"] = dict(path=self.designation_paths[tid].name, sha256=critical.sealing.sha(data)) self.manifest_path.write_bytes(critical.sealing.json_bytes(self.manifest)) def build(self, seed=0, raters=None, coordinator=None): critical.build(self.manifest_path, raters or self.raters, coordinator or self.coordinator, self.f.registry_path, seed) return json.loads(((coordinator or self.coordinator)/"critical-key.json").read_bytes()) def form_path(self, alias): return self.raters/alias/"rating-form.json" def fill(self, key, alias, judgments, adjudicate=False): path = self.form_path(alias) form = json.loads(path.read_bytes()) for row, pair in zip(form["ratings"], judgments): if pair is None: continue row.update(violation=pair[0], invented_restriction=pair[1], reason="Artificial binary fixture only, no substantive judgment.", artifact_scope="Entire artificial artifact reviewed; no actual study assessment.", artifact_evidence=[], completed_at_utc=self.now()) if adjudicate: basis = {} for rater in key["independent_rater_aliases"].values(): original = json.loads(self.form_path(rater).read_bytes()) match = next(r for r in original["ratings"] if r["artifact_id"] == row["artifact_id"] and r["requirement_id"] == row["requirement_id"]) basis[rater] = {k: match[k] for k in ("violation", "invented_restriction")} row.update(independent_judgments=basis, resolution_note="Artificial resolution fixture only.") form["locked_at_utc"] = self.now() path.write_bytes(critical.sealing.json_bytes(form)) return path def summary(self, paths=()): return critical.summarize(self.key_path, paths, self.f.registry_path) def test_blinded_bytes_public_designation_and_blank_forms(self): key = self.build() for spec in key["forms"]: root = self.raters/spec["assessor_id"] for artifact in key["artifacts"]: original = next(r for r in self.manifest["artifacts"] if r["run_id"] == artifact["source_run_id"]) self.assertEqual((root/"artifacts"/(artifact["artifact_id"]+".txt")).read_bytes(), self.f.contents[original["artifact_id"]]) self.assertEqual((root/"public"/(artifact["task_id"]+".txt")).read_bytes(), self.public[artifact["task_id"]].encode()) public = root/"critical-requirements"/(artifact["task_id"]+".json") self.assertEqual(critical.sealing.sha(public.read_bytes()), artifact["designation_sha256"]) for path in root.rglob("*"): if path.is_file() and "artifacts" not in path.parts: content = path.read_bytes() for forbidden in (b"condition_id", b"fixture-condition", b"fixture-run-", b"private-human-designator", b"artificial_configuration"): self.assertNotIn(forbidden, content) form = json.loads((root/"rating-form.json").read_bytes()) self.assertTrue(all(r["violation"] is None and r["invented_restriction"] is None for r in form["ratings"])) def test_counts_denominators_invented_restrictions_and_separate_series(self): key = self.build() a, b = key["independent_rater_aliases"].values() p1 = self.fill(key, a, [(1, 1), (0, 0)]) p2 = self.fill(key, b, [(0, 0), (0, 0)]) report = self.summary([p1, p2]) group = report["independent"][a]["overall"] self.assertEqual((group["violated_critical_requirements"], group["assessed_critical_requirements"], group["critical_requirement_violation_rate"]), (1, 2, .5)) self.assertEqual(group["artifact_violation_rate"], dict(numerator=1, denominator=2, rate=.5)) self.assertEqual(group["requirements_with_assessed_invented_restriction"], 1) self.assertEqual(group["artifacts_with_assessed_invented_restriction"], 1) self.assertIsNone(report["adjudicated"]["overall"]["critical_requirement_violation_rate"]) agreement = report["independent_rater_agreement"]["overall"] self.assertEqual(agreement["jointly_assessed_requirement_denominator"], 2) self.assertEqual(agreement["dimensions"]["violation"]["raw_agreement"], .5) self.assertEqual(agreement["dimensions"]["invented_restriction"]["raw_agreement"], .5) def test_partial_positive_is_lower_bound_and_missing_not_pass_or_failure(self): key = self.build() a, b = key["independent_rater_aliases"].values() p1 = self.fill(key, a, [(1, 0), None]) p2 = self.fill(key, b, [(1, 0), (0, 0)]) report = self.summary([p1, p2]) group = report["independent"][a]["overall"] self.assertEqual((group["assessed_critical_requirements"], group["missing_judgments"], group["artifacts_with_assessed_critical_violation"]), (1, 1, 1)) self.assertIsNone(group["critical_requirement_violation_rate"]) self.assertIsNone(group["artifact_violation_rate"]["rate"]) self.assertEqual(report["independent_rater_agreement"]["overall"]["jointly_assessed_requirement_denominator"], 1) def test_reviewed_empty_vs_unfinished_and_unavailable(self): original = copy.deepcopy(self.designations[self.tid]) for state in ("reviewed", "unfinished", "unavailable"): with self.subTest(state=state): self.designations[self.tid] = dict(original, review_status="unfinished" if state == "unfinished" else "reviewed", requirements=[], completed_at_utc=None if state == "unfinished" else original["completed_at_utc"]) self.save() if state == "unavailable": self.manifest["designations"][0].update(designation=None, unavailability_reason="Artificial unavailable designation") self.manifest_path.write_bytes(critical.sealing.json_bytes(self.manifest)) raters, coordinator = self.f.root/(state+"-raters"), self.f.root/(state+"-key") self.build(raters=raters, coordinator=coordinator) group = critical.summarize(coordinator/"critical-key.json", [], self.f.registry_path)["adjudicated"]["overall"] self.assertEqual(group["reviewed_empty_designation_artifacts"], 2 if state == "reviewed" else 1) self.assertEqual(group["declared_coverage_complete"], state == "reviewed") self.assertIsNone(group["critical_requirement_violation_rate"]) self.assertIsNone(group["judgment_coverage"]["rate"]) if state == "unavailable": self.assertIsNone(group["artifacts"][0]["designated_requirements"]) def test_unfinished_designation_stays_incomplete_even_when_listed_items_rated(self): self.designations[self.tid].update(review_status="unfinished", completed_at_utc=None) self.save() key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, [(0, 0), (0, 0)]) group = self.summary([path])["independent"][a]["overall"] self.assertEqual(group["missing_judgments"], 0) self.assertFalse(group["declared_coverage_complete"]) self.assertIsNone(group["critical_requirement_violation_rate"]) def test_designation_shared_for_same_task_across_conditions(self): self.f.source["runs"][1]["task_id"] = self.tid self.f.output = self.f.root/"shared-task-archive" self.f.seal() commit = json.loads((self.f.output/"packet-build-manifest.json").read_bytes()) self.manifest["archives"][0]["path"] = self.f.output.name self.manifest["artifacts"][1]["task_id"] = self.tid for artifact in self.manifest["artifacts"]: artifact["seal_record_id"] = commit["seal_record_id"] self.manifest["designations"] = self.manifest["designations"][:1] self.save() key = self.build() self.assertEqual(len({a["designation_sha256"] for a in key["artifacts"]}), 1) self.assertEqual([len(a["requirements"]) for a in key["artifacts"]], [2, 2]) def test_duplicate_and_unknown_requirements_and_hidden_fields_rejected(self): original = copy.deepcopy(self.designations[self.tid]) for change, error in ((lambda d: d["requirements"].append(copy.deepcopy(d["requirements"][0])), "duplicate designated requirement"), (lambda d: d["requirements"][0].update(requirement_id="R99"), "unknown public requirement"), (lambda d: d["requirements"][0].update(extra_rule="hidden"), "unexpected/missing fields")): self.designations[self.tid] = copy.deepcopy(original) change(self.designations[self.tid]) self.save() with self.assertRaisesRegex(critical.sealing.ValidationError, error): self.build() self.assertFalse(self.raters.exists()) def test_invalid_requirement_source_locations_and_quotes_rejected(self): original = copy.deepcopy(self.designations[self.tid]) changes = ( (lambda d: d["requirements"][0]["requirement_location"].update(start=True), "invalid span bounds"), (lambda d: d["requirements"][0].update(requirement_location=copy.deepcopy(d["requirements"][1]["requirement_location"])), "whole matching public requirement"), (lambda d: d["requirements"][0]["source_locations"][0].update(source_id="S99"), "unknown authorized source"), (lambda d: d["requirements"][0]["source_locations"][0].update(source_id="S2"), "outside named source section"), (lambda d: d["requirements"][0]["source_locations"][0].update(text="wrong quote"), "quote does not match"), (lambda d: d["requirements"][0]["source_locations"].append(copy.deepcopy(d["requirements"][0]["source_locations"][0])), "duplicate public source"), ) for change, error in changes: self.designations[self.tid] = copy.deepcopy(original) change(self.designations[self.tid]) self.save() with self.assertRaisesRegex(critical.sealing.ValidationError, error): self.build() def test_source_hash_task_version_and_file_hash_mismatches_rejected(self): original = copy.deepcopy(self.designations[self.tid]) for change, error in ((lambda d: d.update(source_packet_sha256="0"*64), "identity/hash/version"), (lambda d: d.update(rubric_version="2.0"), "identity/hash/version"), (lambda d: d.update(designation_version="2.0"), "incompatible")): self.designations[self.tid] = copy.deepcopy(original) change(self.designations[self.tid]) self.save() with self.assertRaisesRegex(critical.sealing.ValidationError, error): self.build() self.designations[self.tid] = original self.save() self.manifest["designations"][0]["designation"]["sha256"] = "0"*64 self.manifest_path.write_bytes(critical.sealing.json_bytes(self.manifest)) with self.assertRaisesRegex(critical.sealing.ValidationError, "source hash mismatch"): self.build() def test_adjudication_requires_both_original_binary_pairs_and_stays_separate(self): key = self.build() a, b = key["independent_rater_aliases"].values() p1 = self.fill(key, a, [(1, 1), (0, 0)]) p2 = self.fill(key, b, [(0, 0), (0, 0)]) pa = self.fill(key, key["adjudicator_alias"], [(0, 0), (0, 0)], adjudicate=True) with self.assertRaisesRegex(critical.sealing.ValidationError, "both independent"): self.summary([p1, pa]) report = self.summary([pa, p2, p1]) self.assertEqual(report["adjudicated"]["overall"]["critical_requirement_violation_rate"], 0) self.assertEqual(report["independent"][a]["overall"]["critical_requirement_violation_rate"], .5) form = json.loads(pa.read_bytes()) form["ratings"][0]["independent_judgments"][a]["invented_restriction"] = 0 pa.write_bytes(critical.sealing.json_bytes(form)) with self.assertRaisesRegex(critical.sealing.ValidationError, "stale adjudication"): self.summary([p1, p2, pa]) def test_binary_flag_relationship_duplicates_and_extra_method_fields_rejected(self): key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, [(1, 1), (0, 0)]) original = path.read_bytes() with self.assertRaisesRegex(critical.sealing.ValidationError, "duplicate assessment series"): self.summary([path, path]) for change, error in ((lambda f: f["ratings"].append(copy.deepcopy(f["ratings"][0])), "duplicate assessed requirement"), (lambda f: f["ratings"][0].update(violation=True), "expected integer"), (lambda f: f["ratings"][0].update(violation=0, invented_restriction=1), "requires an assessed violation"), (lambda f: f["ratings"][0].update(invented_restriction=None), "complete both binary fields"), (lambda f: f["ratings"][0].update(condition_id="excluded"), "unexpected/missing fields"), (lambda f: f.update(critical_form_version="2.0"), "incompatible")): form = json.loads(original) change(form) path.write_bytes(critical.sealing.json_bytes(form)) with self.assertRaisesRegex(critical.sealing.ValidationError, error): self.summary([path]) def test_artifact_evidence_identity_and_judgment_timing(self): key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, [(1, 0), (0, 0)]) form = json.loads(path.read_bytes()) form["ratings"][0]["artifact_evidence"] = [dict(start=0, end=10, text="Artificial")] path.write_bytes(critical.sealing.json_bytes(form)) self.summary([path]) # A matched quote is permitted, not semantic proof. original = path.read_bytes() for change, error in ((lambda f: f["ratings"][0].update(artifact_evidence=[dict(start=0, end=3, text="bad")]), "quote does not match"), (lambda f: f["ratings"][0].update(source_packet_sha256="0"*64), "identity or hash mismatch"), (lambda f: f["ratings"][0].update(artifact_scope=None), "expected nonempty string"), (lambda f: f.update(locked_at_utc=None), "precede form lock"), (lambda f: f["ratings"][0].update(completed_at_utc="2000-01-01T00:00:00Z"), "follow sealing/designation")): form = json.loads(original) change(form) path.write_bytes(critical.sealing.json_bytes(form)) with self.assertRaisesRegex(critical.sealing.ValidationError, error): self.summary([path]) def test_zero_joint_denominator_and_missing_forms_never_imply_agreement(self): self.build() agreement = self.summary()["independent_rater_agreement"]["overall"] self.assertEqual(agreement["jointly_assessed_requirement_denominator"], 0) self.assertEqual(agreement["provided_instances_missing_joint_judgments"], 2) self.assertIsNone(agreement["dimensions"]["violation"]["raw_agreement"]) self.assertIsNone(agreement["dimensions"]["invented_restriction"]["raw_agreement"]) def test_source_section_ends_before_later_non_source_heading(self): packet = self.public[self.tid]+"\r\n## Private notes are not S2\r\nUnrelated text.\r\n" _, sources = critical.public_catalog(packet, self.tid) self.assertEqual(sources["S2"][1], packet.index("## Private notes")) def test_duplicate_task_designations_and_unsealed_inputs_rejected(self): self.manifest["designations"].append(copy.deepcopy(self.manifest["designations"][0])) self.save() with self.assertRaisesRegex(critical.sealing.ValidationError, "duplicate task designation"): self.build() self.manifest["designations"].pop() self.save() (self.f.output/"packet-build-manifest.json").unlink() with self.assertRaisesRegex(critical.sealing.ValidationError, "unsealed"): self.build() def test_cached_designations_allow_removed_originals_and_reject_corruption(self): key = self.build() self.manifest_path.unlink() for path in self.designation_paths.values(): path.unlink() self.summary() path = self.coordinator/key["designation_copies"][0]["path"] path.write_bytes(path.read_bytes()+b"corruption") with self.assertRaisesRegex(critical.sealing.ValidationError, "source hash mismatch"): self.summary() def test_read_only_inputs_existing_output_refusal_and_cli(self): def snapshot(root): return {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file()} before = snapshot(self.f.output) with self.assertRaisesRegex(critical.sealing.ValidationError, "outside sealed archives"): self.build(raters=self.f.output/"new-raters") with self.assertRaisesRegex(critical.sealing.ValidationError, "disjoint"): self.build(raters=self.f.root/"same", coordinator=self.f.root/"same") key = self.build() rater_bytes, coordinator_bytes = snapshot(self.raters), snapshot(self.coordinator) with self.assertRaisesRegex(critical.sealing.ValidationError, "existing output"): self.build() stdout, stderr = io.StringIO(), io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = critical.main(["summarize", "--key", str(self.key_path), "--registry", str(self.f.registry_path)]) self.assertEqual(code, 0) self.assertEqual(json.loads(stdout.getvalue())["critical_report_version"], "1.0") self.assertEqual(before, snapshot(self.f.output)) self.assertEqual(rater_bytes, snapshot(self.raters)) self.assertEqual(coordinator_bytes, snapshot(self.coordinator)) def test_failed_writes_no_success_record_and_duplicate_json_rejection(self): stdout, stderr = io.StringIO(), io.StringIO() with mock.patch.object(critical.packets, "write_bytes", side_effect=OSError("artificial write failure")): with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = critical.main(["build", "--manifest", str(self.manifest_path), "--rater-output", str(self.raters), "--coordinator-output", str(self.coordinator), "--registry", str(self.f.registry_path)]) self.assertEqual(code, 2) self.assertFalse(self.key_path.exists()) self.assertEqual(stdout.getvalue(), "") with self.assertRaisesRegex(critical.sealing.ValidationError, "duplicate JSON key"): critical.sealing.parse_json(b'{"requirement_id":"R1","requirement_id":"R2"}', "artificial duplicate") if __name__ == "__main__": unittest.main()