"""Temporary artificial parser/archive/numeric fixtures; no study observations.""" import copy import json import unittest from unittest import mock import audit_boundary as audit import initial_contract as initial import test_b1_contract as fixtures class InitialAuditTests(unittest.TestCase): def setUp(self): self.base = fixtures.B1AuditTests() self.base.setUp() self.addCleanup(self.base.doCleanups) self.f, self.sealer = self.base.f, self.base.sealer self.task_packet = self.base.values["TASK_PACKET_JSON"] self.content = self.render(self.task_packet) self.f.manifest["audit_manifest_version"] = "1.1" for index, (row, run) in enumerate(zip(self.f.manifest["runs"], self.sealer.source["runs"])): record_id = "fixture-prompt-"+("a" if index == 0 else "b") self.base.replace_source(record_id, self.content) self.f.packet_bytes[record_id] = self.content slots = (copy.deepcopy(row["request_slots"]) if index else [dict(attempt_id=a["attempt_id"], kind="initial") for a in run["attempts"] if a["role"] == "worker"]) initial_slots = [] for sequence, slot in enumerate((s for s in slots if s["kind"] == "initial"), 1): attempt = next(a for a in run["attempts"] if a["attempt_id"] == slot["attempt_id"]) if record_id not in attempt["record_ids"]: attempt["record_ids"].append(record_id) initial_slots.append(dict(input_scope="initial", packet_id="initial-"+slot["attempt_id"], attempt_id=slot["attempt_id"], sequence=sequence, record_id=record_id, sha256=audit.sealing.sha(self.content))) continuation_slots = [dict(p, input_scope="continuation", sequence=len(initial_slots)+n) for n, p in enumerate(row["packets"], 1)] converted = dict(**{k: row[k] for k in (*audit.IDENTITY, "archive_id", "seal_record_id")}, audit_scope="combined", request_slots=slots, initial_input=dict(contract=initial.CONTRACT, packet_inventory_complete=True), continuation_input=dict(contract=row["contract"], packet_inventory_complete=row["packet_inventory_complete"]), packets=initial_slots+continuation_slots) self.f.manifest["runs"][index] = converted self.base.reseal() def render(self, value): return initial.TEMPLATE_PATH.read_bytes().decode().replace("{{TASK_PACKET_JSON}}", json.dumps(value, ensure_ascii=False)).encode() def validate(self, content): return initial.validate_input(content, self.base.bounds, self.base.tid, self.base.public_hash) def completed(self, value=0, filename="independent-1.blank.json"): form = self.f.completed(value, filename) for row, run in zip(form["ratings"], self.f.manifest["runs"]): row["reviewed_packet_ids"] = [p["packet_id"] for p in run["packets"] if p["record_id"] is not None] return form def select_scope(self, scope): for row in self.f.manifest["runs"]: row["audit_scope"] = scope if scope == "initial-only": row["continuation_input"] = None row["packets"] = [p for p in row["packets"] if p["input_scope"] == "initial"] elif scope == "continuation-only": row["initial_input"] = None row["packets"] = [p for p in row["packets"] if p["input_scope"] == "continuation"] if row["continuation_input"]["contract"] == audit.CONTRACT: row["request_slots"] = None def result(self, form=None): paths = [] if form is None else [self.f.write_form(form)] return self.f.summarize(paths) def test_literal_wrapper_single_json_string_and_exact_packet(self): self.assertEqual(self.validate(self.content), self.task_packet) ascii_content = initial.TEMPLATE_PATH.read_bytes().decode().replace("{{TASK_PACKET_JSON}}", json.dumps(self.task_packet)).encode() self.assertEqual(self.validate(ascii_content), self.task_packet) self.assertIn("\r\n", self.validate(self.content)) def test_invalid_value_types_and_additional_fields_or_blocks_rejected(self): for value in ({"task_packet": self.task_packet}, [self.task_packet], 1, True, None): with self.subTest(value_type=type(value).__name__): with self.assertRaises(ValueError): self.validate(self.render(value)) for bad in (b"Extra instruction\n"+self.content, self.content+b"Extra instruction\n", self.content+b"\nCURRENT_ARTIFACT_JSON\n{}\n", self.content.replace(b"Produce the complete", b"Evaluate the complete", 1), self.content[:-1], self.content.replace(b"IMMUTABLE_TASK_PACKET_JSON", b"TASK_PACKET_JSON", 1)): with self.assertRaises(ValueError): self.validate(bad) def test_hash_identity_utf8_placeholder_and_byte_bound_rejections(self): for invalid in (self.render(self.task_packet+"\nArtificial extra source."), self.content+b"\xff", self.render("{{UNRESOLVED}}")): with self.assertRaises(ValueError): self.validate(invalid) with self.assertRaises(ValueError): initial.validate_input(self.content, self.base.bounds, "wrong-task", self.base.public_hash) with self.assertRaisesRegex(ValueError, "byte bound"): initial.validate_input(self.content, dict(max_input_bytes=1), self.base.tid, self.base.public_hash) with mock.patch.object(initial, "TEMPLATE_SHA256", "0"*64): with self.assertRaisesRegex(ValueError, "template hash"): self.validate(self.content) def test_combined_preparation_preserves_bytes_and_form_interface(self): context = self.f.prepare() self.assertEqual(context["audit_context_version"], "1.1") initial_copies = [c for c in context["copies"] if c["kind"] == "common_initial_input"] self.assertEqual(len(initial_copies), 3) for copy_row in initial_copies: self.assertEqual((self.f.output/copy_row["path"]).read_bytes(), self.content) form = self.f.form() self.assertEqual(form["audit_form_version"], "1.0") self.assertTrue(all(set(row) == set(audit.ROW_FIELDS) and row["violation"] is None for row in form["ratings"])) report = self.result() self.assertEqual(report["audit_report_version"], "1.1") self.assertFalse(report["whole_run_compliance_certified"]) self.assertFalse(report["semantic_absence_of_restricted_information_certified"]) def test_combined_complete_review_and_condition_coverage(self): self.f.prepare() report = self.result(self.completed()) data = report["series"]["independent:artificial-reviewer-a"] self.assertEqual(data["overall"]["counts"]["compliant"], 2) self.assertEqual(data["overall"]["assigned_run_denominator"], 2) for run in data["runs"]: self.assertEqual(run["audit_scope"], "combined") self.assertFalse(run["whole_run_compliance_certified"]) for coverage in run["input_scope_coverage"].values(): self.assertTrue(coverage["review_coverage_complete"]) for condition in ("C", "B1"): self.assertEqual(data["by_condition"][condition]["audit_scope_counts"]["combined"], 1) self.assertEqual(report["series"]["adjudicated"]["overall"]["counts"]["unassessed"], 2) empty = data["by_audit_scope"]["initial-only"]["input_scope_coverage"]["initial"] self.assertEqual(empty["declared_attempt_denominator"], 0) self.assertIsNone(empty["packet_review_fraction"]) def test_initial_only_never_implies_continuation_or_whole_run_compliance(self): self.select_scope("initial-only") self.f.prepare() report = self.result(self.completed()) data = report["series"]["independent:artificial-reviewer-a"] self.assertEqual(data["overall"]["counts"]["compliant"], 2) for run in data["runs"]: self.assertEqual(run["audit_scope"], "initial-only") self.assertFalse(run["input_scope_coverage"]["continuation"]["in_scope"]) self.assertIsNone(run["input_scope_coverage"]["continuation"]["packet_inventory_complete"]) self.assertFalse(run["whole_run_compliance_certified"]) self.assertEqual(data["overall"]["input_scope_coverage"]["continuation"]["out_of_scope_run_count"], 2) def test_continuation_only_does_not_validate_or_claim_initial_inputs(self): self.select_scope("continuation-only") # Deliberately invalid initial wrapper is archived, but not declared for audit. self.base.replace_source("fixture-prompt-b", b"Artificial non-contract initial bytes.\n") self.base.reseal() self.f.prepare() data = self.result(self.completed())["series"]["independent:artificial-reviewer-a"] self.assertEqual(data["overall"]["counts"]["compliant"], 2) for run in data["runs"]: self.assertEqual(run["audit_scope"], "continuation-only") self.assertFalse(run["input_scope_coverage"]["initial"]["in_scope"]) self.assertFalse(run["whole_run_compliance_certified"]) def test_missing_initial_packet_keeps_combined_unknown_with_continuation_reviewed(self): self.f.manifest["runs"][1]["packets"][0].update(record_id=None, sha256=None) self.f.prepare() report = self.result(self.completed()) data = report["series"]["independent:artificial-reviewer-a"] run = data["runs"][1] self.assertEqual(run["status"], "unassessed") self.assertEqual(run["input_scope_coverage"]["initial"]["missing_packet_count"], 1) self.assertFalse(run["input_scope_coverage"]["initial"]["review_coverage_complete"]) self.assertTrue(run["input_scope_coverage"]["continuation"]["review_coverage_complete"]) self.assertEqual(data["overall"]["assigned_run_denominator"], 2) def test_missing_continuation_does_not_hide_completed_initial_review(self): self.f.manifest["runs"][1]["packets"][-1].update(record_id=None, sha256=None) self.f.prepare() run = self.result(self.completed())["series"]["independent:artificial-reviewer-a"]["runs"][1] self.assertEqual(run["status"], "unassessed") self.assertTrue(run["input_scope_coverage"]["initial"]["review_coverage_complete"]) self.assertFalse(run["input_scope_coverage"]["continuation"]["review_coverage_complete"]) def test_incomplete_initial_inventory_or_missing_judgment_stays_unknown(self): self.f.manifest["runs"][1]["initial_input"]["packet_inventory_complete"] = False self.f.prepare() report = self.result(self.completed()) self.assertEqual(report["series"]["independent:artificial-reviewer-a"]["runs"][1]["status"], "unassessed") self.assertEqual(report["series"]["independent:artificial-reviewer-b"]["overall"]["counts"]["unassessed"], 2) def test_initial_attempt_designations_and_failed_packet_slots_cannot_be_omitted(self): original = copy.deepcopy(self.f.manifest) for target in ("designation", "packet", "scope"): with self.subTest(target=target): row = self.f.manifest["runs"][0] if target == "designation": row["request_slots"].pop(0) elif target == "packet": row["packets"].pop(0) else: row["packets"][0]["input_scope"] = "continuation" with self.assertRaises(ValueError): self.f.prepare() self.assertFalse(self.f.output.exists()) self.f.manifest = copy.deepcopy(original) def test_initial_transport_retry_identity_kept_despite_archive_array_order(self): run = self.sealer.source["runs"][1] retry = dict(run["attempts"][0], attempt_id="artificial-initial-retry", retry_of="fixture-request-b", status="failed") run["attempts"].append(retry) # Later array position is not a stage designation. row = self.f.manifest["runs"][1] row["request_slots"].append(dict(attempt_id=retry["attempt_id"], kind="initial")) row["packets"][-1]["sequence"] = 3 row["packets"].insert(1, dict(row["packets"][0], packet_id="initial-retry-packet", attempt_id=retry["attempt_id"], sequence=2)) self.base.reseal() self.f.prepare() result = self.result(self.completed())["series"]["independent:artificial-reviewer-a"] self.assertEqual(result["overall"]["assigned_run_denominator"], 2) self.assertEqual(result["runs"][1]["input_scope_coverage"]["initial"]["declared_attempt_count"], 2) def test_confirmed_initial_violation_survives_missing_continuation_coverage(self): self.f.manifest["runs"][1]["packets"][-1].update(record_id=None, sha256=None) self.f.prepare() report = self.result(self.completed(1)) run = report["series"]["independent:artificial-reviewer-a"]["runs"][1] self.assertEqual(run["status"], "noncompliant") self.assertFalse(run["complete_run_review"]) self.assertEqual(run["input_scope_coverage"]["continuation"]["missing_packet_count"], 1) def test_initial_evidence_hash_and_locations_use_existing_human_validation(self): self.f.prepare() for target in ("hash", "location"): form = self.completed(1) finding = form["ratings"][0]["findings"][0] if target == "hash": finding["packet_sha256"] = "0"*64 else: finding["location"]["text"] = "wrong" with self.assertRaises(ValueError): self.result(form) def test_unknown_contract_scope_and_out_of_scope_declarations_rejected(self): original = copy.deepcopy(self.f.manifest) for target in ("contract", "scope", "null", "extra", "legacy-version"): with self.subTest(target=target): row = self.f.manifest["runs"][0] if target == "contract": row["initial_input"]["contract"] = "common-initial-worker/2.0" elif target == "scope": row["audit_scope"] = "whole-run" elif target == "null": row["audit_scope"] = "continuation-only" elif target == "extra": row["initial_input"]["worker_report"] = None else: self.f.manifest["audit_manifest_version"] = "1.0" with self.assertRaises(ValueError): self.f.prepare() self.f.manifest = copy.deepcopy(original) def test_declared_bad_initial_wrapper_and_archived_hash_rejected_before_output(self): record = self.f.manifest["runs"][1]["packets"][0] record["sha256"] = "0"*64 with self.assertRaisesRegex(ValueError, "packet hash"): self.f.prepare() bad = self.content+b"Extra diagnostic\n" self.base.replace_source("fixture-prompt-b", bad) record["sha256"] = audit.sealing.sha(bad) self.base.reseal() with self.assertRaisesRegex(ValueError, "literal wrapper"): self.f.prepare() self.assertFalse(self.f.output.exists()) def test_no_overwrites_or_unsealed_archives_and_context_versions_bound(self): self.f.prepare() with self.assertRaises(ValueError): self.f.prepare() context_path = self.f.output/"audit-context.json" context = json.loads(context_path.read_bytes()) context["audit_context_version"] = "1.0" context_path.write_bytes(audit.sealing.json_bytes(context)) with self.assertRaisesRegex(ValueError, "context/manifest version"): self.result() context["audit_context_version"] = "1.1" context_path.write_bytes(audit.sealing.json_bytes(context)) (self.sealer.output/"packet-build-manifest.json").unlink() with self.assertRaisesRegex(ValueError, "unsealed"): self.result() def test_adjudication_and_independent_series_remain_separate(self): self.f.prepare() first = self.completed(0) second = self.completed(1, "independent-2.blank.json") adjud = self.completed(0, "adjudicated.blank.json") for row in adjud["ratings"]: row.update(independent_judgments=[0, 1], resolution_note="Artificial fixture resolution, no study finding.") paths = [self.f.write_form(form, name) for form, name in ((first, "first.json"), (second, "second.json"), (adjud, "adjud.json"))] report = self.f.summarize(paths) self.assertEqual(report["series"]["independent:artificial-reviewer-a"]["overall"]["counts"]["compliant"], 2) self.assertEqual(report["series"]["independent:artificial-reviewer-b"]["overall"]["counts"]["noncompliant"], 2) self.assertEqual(report["series"]["adjudicated"]["overall"]["counts"]["compliant"], 2) self.assertEqual(report["independent_agreement"]["overall"]["assessed_run_pair_denominator"], 2) def test_mixed_declared_scopes_keep_all_runs_and_separate_scope_counts(self): row = self.f.manifest["runs"][0] row.update(audit_scope="initial-only", continuation_input=None) row["packets"] = [p for p in row["packets"] if p["input_scope"] == "initial"] self.f.prepare() data = self.result(self.completed())["series"]["independent:artificial-reviewer-a"] self.assertEqual(data["overall"]["assigned_run_denominator"], 2) self.assertEqual(data["overall"]["audit_scope_counts"], {"initial-only": 1, "continuation-only": 0, "combined": 1}) self.assertEqual(data["overall"]["input_scope_coverage"]["initial"]["in_scope_run_count"], 2) self.assertEqual(data["overall"]["input_scope_coverage"]["continuation"]["out_of_scope_run_count"], 1) self.assertFalse(data["overall"]["whole_run_compliance_certified"]) if __name__ == "__main__": unittest.main()