"""Temporary artificial archival/parser/numeric fixtures, never study observations.""" import copy from datetime import datetime, timezone import json import re import unittest from unittest import mock import audit_boundary as audit import test_seal_runs as sealing_fixtures import test_assemble_observation as observation_fixtures class BoundaryTests(unittest.TestCase): def setUp(self): self.f = sealing_fixtures.SealingTests() self.f.setUp() self.addCleanup(self.f.doCleanups) self.root = self.f.root self.packet_bytes = {} for index, run in enumerate(self.f.source["runs"]): obs = observation_fixtures.ObservationTests() obs.setUp() self.addCleanup(obs.doCleanups) tid = run["task_id"] task_text = obs.task_text.replace("FIXTURE-TASK", tid) obs.manifest["task_packet"] = dict(task_id=tid, **obs.put("public-packet.txt", task_text.encode())) task = self.f.registry["tasks"][index] (self.f.refs/task["public_packet"]["path"]).write_bytes(task_text.encode()) task["public_packet"]["sha256"] = audit.sealing.sha(task_text.encode()) obs.assemble() content = (obs.model/"planner-input.txt").read_bytes() record_id = "artificial-planner-packet-"+str(index) self.packet_bytes[record_id] = content self.f.add_record(record_id, "submitted_prompt", content) run["record_ids"].append(record_id) run["attempts"].append(dict(attempt_id="artificial-planner-attempt-"+str(index), phase="adaptation", role="planner", status="succeeded", retry_of=None, started_at_utc=None, ended_at_utc=None, record_ids=[record_id])) self.f.registry_path.write_bytes(audit.sealing.json_bytes(self.f.registry)) self.f.set_availability() self.f.seal() commit = json.loads((self.f.output/"packet-build-manifest.json").read_bytes()) self.manifest = dict(audit_manifest_version="1.0", audit_id="artificial-audit", archives=[dict(archive_id="archive", path=str(self.f.output))], independent_rater_ids=["artificial-reviewer-a", "artificial-reviewer-b"], adjudicator_id="artificial-adjudicator", bounds=dict(max_input_bytes=2097152, max_memory_entries=32), runs=[]) for index, run in enumerate(self.f.source["runs"]): record_id = "artificial-planner-packet-"+str(index) self.manifest["runs"].append(dict(**{k: run[k] for k in audit.IDENTITY}, archive_id="archive", seal_record_id=commit["seal_record_id"], contract=audit.CONTRACT, packet_inventory_complete=True, packets=[dict(packet_id="p"+str(index), attempt_id="artificial-planner-attempt-"+str(index), sequence=1, record_id=record_id, sha256=audit.sealing.sha(self.packet_bytes[record_id]))])) self.path = self.root/"audit-manifest.json" self.output = self.root/"new-audit" def save(self): self.path.write_bytes(audit.sealing.json_bytes(self.manifest)) def add_retry_and_reseal(self): run = self.f.source["runs"][0] attempt = copy.deepcopy(run["attempts"][-1]) attempt.update(attempt_id="artificial-retry", retry_of=attempt["attempt_id"]) run["attempts"].append(attempt) packet = dict(self.manifest["runs"][0]["packets"][0], packet_id="retry-packet", attempt_id="artificial-retry", sequence=2) self.manifest["runs"][0]["packets"].append(packet) self.f.output = self.root/"second-artificial-archive" self.f.seal() commit = json.loads((self.f.output/"packet-build-manifest.json").read_bytes()) self.manifest["archives"][0]["path"] = str(self.f.output) for row in self.manifest["runs"]: row["seal_record_id"] = commit["seal_record_id"] def prepare(self): self.save() return audit.prepare(self.path, self.output, self.f.registry_path) def form(self, name="independent-1.blank.json"): return json.loads((self.output/name).read_bytes()) def completed(self, value=0, filename="independent-1.blank.json"): form = self.form(filename) stamp = datetime.now(timezone.utc).isoformat() for i, row in enumerate(form["ratings"]): packet = self.manifest["runs"][i]["packets"][0] row.update(violation=value, reviewed_packet_ids=[] if packet["record_id"] is None else [packet["packet_id"]], reason="Artificial review-scope fixture, not a study judgment.", completed_at_utc=stamp) if value == 1: content = self.packet_bytes[packet["record_id"]].decode() row["findings"] = [dict(category="other_restricted_information", packet_id=packet["packet_id"], packet_sha256=packet["sha256"], location=dict(start=0, end=8, text=content[:8]), explanation="Artificial validator evidence fixture, no semantic judgment.")] form["locked_at_utc"] = stamp return form def write_form(self, form, name="completed.json"): path = self.root/name path.write_bytes(audit.sealing.json_bytes(form)) return path def summarize(self, paths=()): return audit.summarize(self.output/"audit-context.json", paths) def test_byte_preserving_copies_blank_forms_and_no_certification(self): context = self.prepare() for item in context["copies"]: if item["kind"] == "planner_input": index = int(item["identity"][1:]) self.assertEqual((self.output/item["path"]).read_bytes(), self.packet_bytes["artificial-planner-packet-"+str(index)]) for filename in ("independent-1.blank.json", "independent-2.blank.json", "adjudicated.blank.json"): form = self.form(filename) self.assertIsNone(form["locked_at_utc"]) self.assertTrue(all(r["violation"] is None and not r["findings"] for r in form["ratings"])) report = self.summarize() self.assertFalse(report["semantic_absence_of_restricted_information_certified"]) self.assertEqual(report["series"]["adjudicated"]["overall"]["counts"]["unassessed"], 2) def test_zero_human_judgments_and_all_run_statuses_retained(self): self.prepare() report = self.summarize([self.write_form(self.completed())]) result = report["series"]["independent:artificial-reviewer-a"] self.assertEqual(result["overall"]["assigned_run_denominator"], 2) self.assertEqual(result["overall"]["counts"], dict(compliant=2, noncompliant=0, unassessed=0)) self.assertEqual({r["declared_run_status"] for r in result["runs"]}, {"incomplete", "completed"}) self.assertEqual(report["series"]["independent:artificial-reviewer-b"]["overall"]["counts"]["unassessed"], 2) def test_structural_pass_does_not_override_human_violation(self): self.prepare() report = self.summarize([self.write_form(self.completed(1))]) result = report["series"]["independent:artificial-reviewer-a"]["overall"] self.assertEqual(result["counts"]["noncompliant"], 2) self.assertEqual(result["confirmed_violation_frequency"]["denominator"], 2) def test_missing_packet_and_incomplete_inventory_stay_unknown(self): self.manifest["runs"][0]["packets"][0].update(record_id=None, sha256=None) self.manifest["runs"][1]["packet_inventory_complete"] = False self.prepare() report = self.summarize([self.write_form(self.completed())]) result = report["series"]["independent:artificial-reviewer-a"]["overall"] self.assertEqual(result["counts"]["unassessed"], 2) self.assertEqual(result["missing_packet_count"], 1) self.assertFalse(result["complete_coverage"]) self.assertFalse(result["confirmed_violation_frequency"]["complete"]) def test_partial_review_zero_stays_unknown_but_confirmed_violation_survives_missing_coverage(self): self.manifest["runs"][0]["packet_inventory_complete"] = False self.prepare() form = self.completed(1) form["ratings"][1].update(violation=0, findings=[], reviewed_packet_ids=[]) report = self.summarize([self.write_form(form)]) result = report["series"]["independent:artificial-reviewer-a"]["overall"] self.assertEqual(result["counts"], dict(compliant=0, noncompliant=1, unassessed=1)) self.assertFalse(result["complete_coverage"]) def test_duplicate_run_or_packet_or_reference_rejected(self): for mutation in ("run", "packet", "attempt", "archive"): with self.subTest(mutation=mutation): original = copy.deepcopy(self.manifest) if mutation == "run": self.manifest["runs"].append(copy.deepcopy(self.manifest["runs"][0])) elif mutation == "packet": self.manifest["runs"][1]["packets"][0]["packet_id"] = "p0" elif mutation == "attempt": self.manifest["runs"][1]["packets"][0]["attempt_id"] = "artificial-planner-attempt-0" else: self.manifest["archives"].append(copy.deepcopy(self.manifest["archives"][0])) with self.assertRaises(ValueError): self.prepare() self.assertFalse(self.output.exists()) self.manifest = original def test_missing_run_slot_or_planner_attempt_rejected(self): for target in ("run", "attempt"): with self.subTest(target=target): original = copy.deepcopy(self.manifest) if target == "run": self.manifest["runs"].pop() else: self.manifest["runs"][0]["packets"] = [] with self.assertRaises(ValueError): self.prepare() self.assertFalse(self.output.exists()) self.manifest = original def test_identity_hash_version_contract_and_numeric_bounds_rejected(self): changes = [("task_id", "wrong"), ("task_version", "2"), ("seal_record_id", "wrong"), ("contract", "unknown"), ("packet_inventory_complete", 1)] for field, value in changes: with self.subTest(field=field): original = copy.deepcopy(self.manifest) self.manifest["runs"][0][field] = value with self.assertRaises(ValueError): self.prepare() self.manifest = original self.manifest["runs"][0]["packets"][0]["sha256"] = "0"*64 with self.assertRaises(ValueError): self.prepare() def test_unsealed_input_rejected(self): (self.f.output/"packet-build-manifest.json").unlink() with self.assertRaisesRegex(ValueError, "unsealed"): self.prepare() def test_structural_validator_reused_and_failure_publishes_nothing(self): with mock.patch.object(audit.response, "input_context", side_effect=ValueError("artificial malformed contract")) as check: with self.assertRaisesRegex(ValueError, "malformed contract"): self.prepare() check.assert_called_once() self.assertFalse(self.output.exists()) def test_actual_invalid_reference_and_wrapper_rejected_by_shared_validator(self): content = self.packet_bytes["artificial-planner-packet-0"] for changed in (content.replace(b'"artifact_id":"a3"', b'"artifact_id":"S1"'), content+b"extra wrapper text\n", content.replace(b'PERSISTENT_MEMORY_JSON\n[]', b'PERSISTENT_MEMORY_JSON\n[{"id":"m1","kind":"observation","statement":"fixture","status":"current","evidence":[{"artifact_id":"unseen","location":"line 1"}]}]')): with self.subTest(changed=changed[-40:]): with self.assertRaises(ValueError): audit.response.input_context(changed, self.manifest["bounds"]) def test_unrelated_metadata_fields_rejected(self): self.manifest["runs"][0]["ratings"] = [] with self.assertRaises(ValueError): self.prepare() def test_invalid_evidence_and_binary_judgments_rejected(self): self.prepare() for field in ("quote", "hash", "packet", "category", "binary", "bool", "duplicate"): with self.subTest(field=field): form = self.completed(1) row = form["ratings"][0] finding = row["findings"][0] if field == "quote": finding["location"]["text"] = "bad" elif field == "hash": finding["packet_sha256"] = "0"*64 elif field == "packet": finding["packet_id"] = "unseen" elif field == "category": finding["category"] = "invented" elif field == "binary": row["violation"] = 2 elif field == "bool": row["violation"] = True else: row["findings"].append(copy.deepcopy(finding)) with self.assertRaises(ValueError): self.summarize([self.write_form(form)]) def test_forms_identity_duplicates_and_stale_hash_rejected(self): self.prepare() original = self.completed() for field in ("identity", "binding", "duplicate", "missing"): with self.subTest(field=field): form = copy.deepcopy(original) if field == "identity": form["ratings"][0]["task_id"] = "wrong" elif field == "binding": form["context_sha256"] = "0"*64 elif field == "duplicate": form["ratings"].append(copy.deepcopy(form["ratings"][0])) else: form["ratings"].pop() with self.assertRaises(ValueError): self.summarize([self.write_form(form)]) path = self.write_form(original) with self.assertRaises(ValueError): self.summarize([path, path]) def test_independent_agreement_and_adjudication_kept_separate(self): self.prepare() first, second = self.completed(0), self.completed(1, "independent-2.blank.json") a, b = self.write_form(first, "a.json"), self.write_form(second, "b.json") adjud = self.completed(0, "adjudicated.blank.json") for row in adjud["ratings"]: row.update(independent_judgments=[0, 1], resolution_note="Artificial resolution fixture.") c = self.write_form(adjud, "c.json") with self.assertRaises(ValueError): self.summarize([c]) report = self.summarize([a, b, c]) agreement = report["independent_agreement"]["overall"] self.assertEqual(agreement["assessed_run_pair_denominator"], 2) self.assertEqual(agreement["raw_agreement"], 0) self.assertEqual(report["series"]["adjudicated"]["overall"]["counts"]["compliant"], 2) self.assertEqual(report["series"]["independent:artificial-reviewer-b"]["overall"]["counts"]["noncompliant"], 2) def test_no_judgment_agreement_denominator_zero(self): self.prepare() agreement = self.summarize()["independent_agreement"]["overall"] self.assertEqual(agreement["assessed_run_pair_denominator"], 0) self.assertIsNone(agreement["raw_agreement"]) self.assertIsNone(audit.group([])["confirmed_violation_frequency"]["rate"]) def test_explicit_empty_inventory_is_not_automatic_compliance(self): run = dict(self.manifest["runs"][0], declared_run_status="failed") run["packets"] = [] self.assertEqual(audit.run_result(run, None)["status"], "unassessed") judgment = dict(violation=0, reviewed_packet_ids=[], findings=[]) self.assertEqual(audit.run_result(run, judgment)["status"], "compliant") run["packet_inventory_complete"] = False self.assertEqual(audit.run_result(run, judgment)["status"], "unassessed") def test_blank_forms_remain_unknown_and_all_series_separate(self): self.prepare() paths = [self.output/name for name in ("independent-1.blank.json", "independent-2.blank.json", "adjudicated.blank.json")] report = self.summarize(paths) for series in report["series"].values(): self.assertEqual(series["overall"]["counts"]["unassessed"], 2) def test_canonical_invalid_reference_excess_history_and_memory_rejected(self): content = self.packet_bytes["artificial-planner-packet-0"].decode() def replace(label, value): return re.sub(r"(?m)^("+label+r"\n)[^\r\n]*", lambda m: m.group(1)+audit.observation.canonical(value), content).encode() memory = [dict(id="m1", kind="observation", statement="Artificial fixture note", status="current", evidence=[dict(artifact_id="unseen", location="line 1")])] with self.assertRaisesRegex(ValueError, "evidence identifier"): audit.response.input_context(replace("PERSISTENT_MEMORY_JSON", memory), self.manifest["bounds"]) memory[0]["evidence"][0]["artifact_id"] = "a3" with self.assertRaisesRegex(ValueError, "memory entries"): audit.response.input_context(replace("PERSISTENT_MEMORY_JSON", memory), dict(max_memory_entries=0)) older = [dict(artifact_id="old"+str(i), text="Artificial earlier fixture") for i in range(3)] with self.assertRaisesRegex(ValueError, "artifact history"): audit.response.input_context(replace("EARLIER_ARTIFACTS_JSON", older), self.manifest["bounds"]) def test_invalid_utf8_duplicate_keys_input_bound_and_chronology_rejected(self): content = self.packet_bytes["artificial-planner-packet-0"] for bad in (content+b"\xff", content.replace(b'"artifact_id":"a3"', b'"artifact_id":"a3","artifact_id":"a4"')): with self.assertRaises(ValueError): audit.response.input_context(bad, self.manifest["bounds"]) for target, value in (("bound", 1), ("bound", True), ("sequence", 0), ("sequence", True)): with self.subTest(target=target, value=value): original = copy.deepcopy(self.manifest) if target == "bound": self.manifest["bounds"]["max_input_bytes"] = value else: self.manifest["runs"][0]["packets"][0]["sequence"] = value with self.assertRaises(ValueError): self.prepare() self.manifest = original def test_identical_retry_input_retains_distinct_attempts_and_one_run_denominator(self): self.add_retry_and_reseal() context = self.prepare() self.assertEqual(sum(c["kind"] == "planner_input" for c in context["copies"]), 3) form = self.completed() form["ratings"][0]["reviewed_packet_ids"].append("retry-packet") result = self.summarize([self.write_form(form)])["series"]["independent:artificial-reviewer-a"]["overall"] self.assertEqual(result["assigned_run_denominator"], 2) self.assertEqual(result["counts"]["compliant"], 2) def test_reversed_packet_order_and_missing_retry_coverage(self): self.add_retry_and_reseal() self.manifest["runs"][0]["packets"].reverse() with self.assertRaisesRegex(ValueError, "chronology"): self.prepare() self.manifest["runs"][0]["packets"].reverse() self.manifest["runs"][0]["packets"][1].update(record_id=None, sha256=None) self.prepare() form = self.completed(1) result = self.summarize([self.write_form(form)])["series"]["independent:artificial-reviewer-a"]["overall"] self.assertEqual(result["counts"]["noncompliant"], 2) self.assertEqual(result["missing_packet_count"], 1) self.assertFalse(result["complete_coverage"]) def test_existing_destinations_and_archives_untouched(self): before = {p.relative_to(self.f.output): p.read_bytes() for p in self.f.output.rglob("*") if p.is_file()} self.prepare() with self.assertRaises(ValueError): self.prepare() report = self.root/"report.json" audit.write_report(self.output/"audit-context.json", [], report) with self.assertRaises(ValueError): audit.write_report(self.output/"audit-context.json", [], report) after = {p.relative_to(self.f.output): p.read_bytes() for p in self.f.output.rglob("*") if p.is_file()} self.assertEqual(before, after) def test_failed_preparation_does_not_publish_context(self): original = audit.packets.write_json def fail(path, value): if path.name == "adjudicated.blank.json": raise OSError("artificial write interruption") return original(path, value) with mock.patch.object(audit.packets, "write_json", side_effect=fail): with self.assertRaises(OSError): self.prepare() self.assertFalse((self.output/"audit-context.json").exists()) def test_failed_report_publication_leaves_no_success_file(self): self.prepare() output = self.root/"new-report.json" with mock.patch("os.link", side_effect=OSError("artificial link failure")): with self.assertRaises(OSError): audit.write_report(self.output/"audit-context.json", [], output) self.assertFalse(output.exists()) def test_tampered_packet_copy_or_manifest_rejected(self): context = self.prepare() copied = self.output/context["copies"][0]["path"] original = copied.read_bytes() copied.write_bytes(original+b"tamper") with self.assertRaises(ValueError): self.summarize() copied.write_bytes(original) self.path.write_bytes(self.path.read_bytes()+b" ") with self.assertRaises(ValueError): self.summarize() if __name__ == "__main__": unittest.main()