"""Temporary artificial audit fixtures only; no task answers or study findings.""" import contextlib import copy from datetime import datetime, timezone import io import json from pathlib import Path import sys import unittest import audit_memory as audit sys.path.insert(0, str(Path(__file__).resolve().parent.parent/"run-sealing")) import test_seal_runs as fixtures class MemoryAuditTests(unittest.TestCase): def setUp(self): self.f = fixtures.SealingTests() self.f.setUp() self.addCleanup(self.f.doCleanups) self.manifest_path = self.f.root/"audit.json" self.forms = self.f.root/"blank-audit-forms" self.memory = [self.entry("entry-a"), self.entry("entry-b")] self.available_b = False def entry(self, identity): return dict(id=identity, kind="observation", statement="Artificial memory string; no task answer.", evidence=[dict(requirement_id="R1", location="Artificial location")], status="current") def prepare(self, available_b=False, memory=None): if memory is not None: self.memory = memory f = self.f # No accounting observations invented; explicitly unavailable in fixture. f.source["records"] = [r for r in f.source["records"] if r["record_id"] != "fixture-usage"] for run in f.source["runs"]: run["record_ids"].remove("fixture-usage") memory_path = f.root/"fixture-memory.source" memory_path.write_bytes(audit.sealing.json_bytes(self.memory)) if available_b: f.add_record("empty-memory", "memory_snapshot", b"[]\r\n") f.source["runs"][1]["record_ids"].append("empty-memory") f.source["conditions"].append(dict(condition_id="second-condition", repetition_ids=["fixture-repetition"])) f.source["runs"][1]["condition_id"] = "second-condition" f.set_availability() for run in f.source["runs"]: run["record_availability"]["usage_records"] = dict(state="unavailable", note="Artificial unavailable usage") f.seal() packet = json.loads((f.output/"packet-build-manifest.json").read_bytes()) self.manifest = dict(audit_manifest_version="1.0", audit_id="artificial-memory-audit", independent_rater_ids=["audit-rater-a", "audit-rater-b"], adjudicator_id="audit-adjudicator", archives=[dict(archive_id="archive-a", path=str(f.output))], runs=[]) for index, run in enumerate(f.source["runs"]): row = {k: run[k] for k in ("run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id")} sid = "fixture-memory" if index == 0 else ("empty-memory" if available_b else None) snapshot = None if sid is None else dict(record_id=sid, sha256=audit.sealing.sha((f.root/(sid+".source")).read_bytes()), selection_basis="Explicit final snapshot in artificial fixture") row.update(archive_id="archive-a", seal_record_id=packet["seal_record_id"], final_memory=snapshot, unavailability_reason="Final memory not available in artificial fixture" if sid is None else None) self.manifest["runs"].append(row) self.save_manifest() def save_manifest(self): self.manifest_path.write_bytes(audit.sealing.json_bytes(self.manifest)) def make_forms(self): audit.write_forms(self.manifest_path, self.forms, self.f.registry_path) return [self.forms/name for name in ("independent-1.json", "independent-2.json", "adjudication.json")] def fill(self, path, judgments): form = json.loads(path.read_bytes()) for row, judgment in zip(form["records"], judgments): if judgment is None: continue row.update(judgment=judgment, rationale="Artificial classification fixture only.", evidence_locations=["Artificial public requirement R1 and memory location"], reviewed_at_utc=datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")) path.write_bytes(audit.sealing.json_bytes(form)) return form def summary(self, paths=()): return audit.summarize(self.manifest_path, paths, self.f.registry_path) def test_blank_forms_do_not_infer_support_from_valid_references(self): self.prepare() paths = self.make_forms() self.assertTrue(all(row["judgment"] is None for path in paths for row in json.loads(path.read_bytes())["records"])) report = self.summary(paths) group = report["independent"]["audit-rater-a"]["overall"] self.assertEqual(group["assessed_memory_entries"], 0) self.assertEqual(group["missing_judgment_entries"], 2) self.assertEqual(group["unavailable_memory_runs"], 1) self.assertIsNone(group["unsupported_normative_rate"]["rate"]) self.assertFalse(group["complete_declared_run_coverage"]) self.assertTrue(all(r["affected"] is None for r in group["runs"])) def test_independent_and_adjudicated_series_and_partial_coverage(self): self.prepare(available_b=True) p1, p2, pa = self.make_forms() self.fill(p1, ["unsupported_normative", "not_normative"]) self.fill(p2, ["supported_normative", None]) report = self.summary([p1, p2]) a = report["independent"]["audit-rater-a"]["overall"] b = report["independent"]["audit-rater-b"]["overall"] self.assertEqual(a["unsupported_normative_rate"], dict(numerator=1, denominator=2, rate=.5)) self.assertEqual(a["confirmed_affected_runs"], 1) self.assertEqual(a["affected_run_rate_over_all_declared"]["rate"], .5) self.assertEqual(a["empty_memory_runs"], 1) self.assertEqual(b["missing_judgment_entries"], 1) self.assertIsNone(b["affected_run_rate_over_all_declared"]["rate"]) self.assertIsNone(b["runs"][0]["affected"]) self.assertEqual(report["adjudicated"]["overall"]["assessed_memory_entries"], 0) empty = report["independent"]["audit-rater-a"]["by_condition"]["second-condition"] self.assertTrue(empty["complete_declared_run_coverage"]) self.assertIsNone(empty["unsupported_normative_rate"]["rate"]) self.assertEqual(empty["affected_run_rate_over_all_declared"]["rate"], 0) def test_adjudication_requires_both_ratings_and_exact_basis(self): self.prepare(available_b=True) p1, p2, pa = self.make_forms() self.fill(p1, ["unsupported_normative", "not_normative"]) self.fill(p2, ["supported_normative", "not_normative"]) adjudication = self.fill(pa, ["supported_normative", "not_normative"]) for index, row in enumerate(adjudication["records"]): row["independent_judgments"] = {"audit-rater-a": "unsupported_normative" if index == 0 else "not_normative", "audit-rater-b": "supported_normative" if index == 0 else "not_normative"} row["resolution_note"] = "Artificial resolution fixture, no substantive assessment." pa.write_bytes(audit.sealing.json_bytes(adjudication)) with self.assertRaisesRegex(audit.sealing.ValidationError, "both independent"): self.summary([pa, p1]) report = self.summary([pa, p2, p1]) self.assertEqual(report["adjudicated"]["overall"]["unsupported_normative_entries"], 0) self.assertEqual(report["independent"]["audit-rater-a"]["overall"]["unsupported_normative_entries"], 1) adjudication["records"][0]["independent_judgments"]["audit-rater-a"] = "not_normative" pa.write_bytes(audit.sealing.json_bytes(adjudication)) with self.assertRaisesRegex(audit.sealing.ValidationError, "mismatched adjudication"): self.summary([p1, p2, pa]) def test_unavailable_not_empty_and_partial_affected_count_is_lower_bound(self): self.prepare() p1, _, _ = self.make_forms() self.fill(p1, ["unsupported_normative", None]) group = self.summary([p1])["independent"]["audit-rater-a"]["overall"] self.assertEqual(group["empty_memory_runs"], 0) self.assertEqual(group["unavailable_memory_runs"], 1) self.assertEqual(group["confirmed_affected_runs"], 1) self.assertFalse(group["complete_declared_run_coverage"]) self.assertIsNone(group["affected_run_rate_over_all_declared"]["rate"]) self.assertIsNone(group["runs"][1]["entry_count"]) def test_empty_memory_has_null_entry_rate_with_zero_denominator(self): self.prepare(available_b=True, memory=[]) group = self.summary()["adjudicated"]["overall"] self.assertEqual(group["empty_memory_runs"], 2) self.assertEqual(group["missing_judgment_entries"], 0) self.assertIsNone(group["unsupported_normative_rate"]["rate"]) self.assertIsNone(group["judgment_coverage"]["rate"]) self.assertTrue(group["complete_declared_run_coverage"]) def test_duplicate_memory_entries_rejected(self): entry = self.entry("same-id") self.prepare(memory=[entry, copy.deepcopy(entry)]) with self.assertRaisesRegex(audit.typed.AssemblyError, "duplicate/ambiguous"): self.summary() def test_invalid_typed_shape_and_reference_rejected(self): for mutation in (lambda e: e.update(kind="mandatory_rule"), lambda e: e.update(evidence=[dict(worker_report_id="report", location="x")])): entry = self.entry("entry") mutation(entry) with self.subTest(entry=entry): with self.assertRaises(audit.typed.AssemblyError): audit.typed_snapshot(audit.sealing.json_bytes([entry])) def test_duplicate_assessment_entries_and_series_rejected(self): self.prepare() p1, _, _ = self.make_forms() with self.assertRaisesRegex(audit.sealing.ValidationError, "duplicate assessment series"): self.summary([p1, p1]) form = json.loads(p1.read_bytes()) form["records"].append(copy.deepcopy(form["records"][0])) p1.write_bytes(audit.sealing.json_bytes(form)) with self.assertRaisesRegex(audit.sealing.ValidationError, "duplicate assessed entry"): self.summary([p1]) def test_selected_hash_and_assessment_identity_mismatch_rejected(self): self.prepare() p1, _, _ = self.make_forms() original = p1.read_bytes() for field, value in (("snapshot_sha256", "0"*64), ("entry_sha256", "0"*64), ("seal_record_id", "wrong-seal")): with self.subTest(field=field): form = json.loads(original) form["records"][0][field] = value p1.write_bytes(audit.sealing.json_bytes(form)) with self.assertRaisesRegex(audit.sealing.ValidationError, "identity or hash mismatch"): self.summary([p1]) self.manifest["runs"][0]["final_memory"]["sha256"] = "0"*64 self.save_manifest() with self.assertRaisesRegex(audit.sealing.ValidationError, "snapshot hash mismatch"): self.summary() def test_form_version_manifest_binding_and_manual_completion_validation(self): self.prepare() p1, _, _ = self.make_forms() original = self.fill(p1, ["not_normative", None]) changes = ( (lambda f: f.update(assessment_form_version="2.0"), "incompatible"), (lambda f: f.update(audit_manifest_sha256="0"*64), "audit identity/hash mismatch"), (lambda f: f["records"][0].update(judgment=True), "invalid human judgment"), (lambda f: f["records"][0].update(reviewed_at_utc="2000-01-01T00:00:00Z"), "after sealing"), (lambda f: f["records"][0].update(evidence_locations=[]), "human evidence locations"), ) for change, error in changes: with self.subTest(error=error): form = copy.deepcopy(original) change(form) p1.write_bytes(audit.sealing.json_bytes(form)) with self.assertRaisesRegex(audit.sealing.ValidationError, error): self.summary([p1]) # Omitted entry rows are missing judgments, not silently classified. original["records"].pop() p1.write_bytes(audit.sealing.json_bytes(original)) group = self.summary([p1])["independent"]["audit-rater-a"]["overall"] self.assertEqual(group["missing_judgment_entries"], 1) self.assertEqual(group["assessed_memory_entries"], 1) def test_snapshot_must_be_assigned_to_run_and_no_automatic_latest_choice(self): self.prepare(available_b=True) self.manifest["runs"][1]["final_memory"] = copy.deepcopy(self.manifest["runs"][0]["final_memory"]) self.save_manifest() with self.assertRaisesRegex(audit.sealing.ValidationError, "this run's sealed memory record"): self.summary() def test_duplicate_missing_and_incompatible_run_selections_rejected(self): self.prepare() original = copy.deepcopy(self.manifest) for change, error in ((lambda m: m["runs"].append(copy.deepcopy(m["runs"][0])), "duplicate/unknown"), (lambda m: m["runs"].pop(), "every supplied archived run"), (lambda m: m["runs"][0].update(rubric_version="2.0"), "identity/task/rubric mismatch")): self.manifest = copy.deepcopy(original) change(self.manifest) self.save_manifest() with self.assertRaisesRegex(audit.sealing.ValidationError, error): self.summary() def test_unsealed_versions_and_duplicate_json_rejected(self): self.prepare() path = self.f.output/"packet-build-manifest.json" packet = path.read_bytes() path.unlink() with self.assertRaisesRegex(audit.sealing.ValidationError, "unsealed"): self.summary() path.write_bytes(packet) self.manifest["audit_manifest_version"] = "2.0" self.save_manifest() with self.assertRaisesRegex(audit.sealing.ValidationError, "incompatible"): self.summary() with self.assertRaisesRegex(audit.typed.AssemblyError, "duplicate JSON key"): audit.typed_snapshot(b'[{"id":"a","id":"b"}]') def test_blank_form_overwrite_refusal_read_only_and_cli(self): self.prepare() def snapshots(root): return {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file()} original = snapshots(self.f.output) with self.assertRaisesRegex(audit.sealing.ValidationError, "outside sealed archives"): audit.write_forms(self.manifest_path, self.f.output/"new-audit", self.f.registry_path) paths = self.make_forms() original_forms = snapshots(self.forms) with self.assertRaisesRegex(audit.sealing.ValidationError, "existing forms destination"): self.make_forms() stdout, stderr = io.StringIO(), io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = audit.main(["summarize", "--manifest", str(self.manifest_path), "--registry", str(self.f.registry_path), "--assessments", *map(str, paths)]) self.assertEqual(code, 0) self.assertEqual(json.loads(stdout.getvalue())["memory_drift_report_version"], "1.0") self.assertEqual(original, snapshots(self.f.output)) self.assertEqual(original_forms, snapshots(self.forms)) self.assertFalse(any(p.name.startswith("rating-") for p in self.forms.iterdir())) if __name__ == "__main__": unittest.main()