"""Temporary artificial B1 parser/archive fixtures; no study inputs or findings.""" import copy import json import unittest import audit_boundary as audit import b1_contract as b1 import test_audit_boundary as fixtures class B1AuditTests(unittest.TestCase): def setUp(self): self.f = fixtures.BoundaryTests() self.f.setUp() self.addCleanup(self.f.doCleanups) self.sealer = self.f.f source = self.sealer.source for run, condition in zip(source["runs"], ("C", "B1")): run["condition_id"] = condition source["runs"][1]["task_id"] = source["runs"][0]["task_id"] source["conditions"] = [dict(condition_id=c, repetition_ids=["fixture-repetition"]) for c in ("C", "B1")] ledger = json.loads(self.sealer.contents["fixture-usage"]) for row, run in zip(ledger["runs"], source["runs"]): row.update({k: run[k] for k in ("condition_id", "task_id")}) self.replace_source("fixture-usage", audit.sealing.json_bytes(ledger)) rendered = self.f.packet_bytes["artificial-planner-packet-0"].decode() self.values = {placeholder: json.loads(rendered.split(label+"\n", 1)[1].split("\n", 1)[0]) for label, placeholder in audit.response.BLOCKS[:4]} self.tid = source["runs"][0]["task_id"] self.public_hash = self.sealer.registry["tasks"][0]["public_packet"]["sha256"] self.bounds = self.f.manifest["bounds"] continuation = source["runs"][1]["attempts"][-1] continuation.update(role="worker", attempt_id="artificial-b1-continuation") row = self.f.manifest["runs"][1] row.update(contract=b1.CONTRACT, request_slots=[dict(attempt_id="fixture-request-b", kind="initial"), dict(attempt_id="artificial-b1-continuation", kind="continuation")]) row["packets"][0]["attempt_id"] = "artificial-b1-continuation" self.set_b1_packet(self.render()) self.reseal() def replace_source(self, rid, content): (self.sealer.root/(rid+".source")).write_bytes(content) self.sealer.contents[rid] = content def set_b1_packet(self, content): self.replace_source("artificial-planner-packet-1", content) self.f.packet_bytes["artificial-planner-packet-1"] = content self.f.manifest["runs"][1]["packets"][0]["sha256"] = audit.sealing.sha(content) def reseal(self): number = len(list(self.sealer.root.glob("mixed-archive-*"))) self.sealer.output = self.sealer.root/("mixed-archive-"+str(number)) self.sealer.set_availability() self.sealer.seal() commit = json.loads((self.sealer.output/"packet-build-manifest.json").read_bytes()) self.f.manifest["archives"][0]["path"] = str(self.sealer.output) for row, run in zip(self.f.manifest["runs"], self.sealer.source["runs"]): row.update({k: run[k] for k in audit.IDENTITY}) row["seal_record_id"] = commit["seal_record_id"] def render(self, values=None, pretty=False): content = b1.TEMPLATE_PATH.read_text(encoding="utf-8") for key, value in (self.values if values is None else values).items(): content = content.replace("{{"+key+"}}", json.dumps(value, ensure_ascii=False, indent=2 if pretty else None)) return content.encode("utf-8") def validate(self, content): return b1.validate_input(content, self.bounds, self.tid, self.public_hash) def add_retry(self, missing=False): run = self.sealer.source["runs"][1] attempt = dict(run["attempts"][-1], attempt_id="artificial-b1-retry", retry_of="artificial-b1-continuation", status="failed") run["attempts"].append(attempt) row = self.f.manifest["runs"][1] row["request_slots"].append(dict(attempt_id=attempt["attempt_id"], kind="continuation")) packet = dict(row["packets"][0], packet_id="b1-retry-packet", attempt_id=attempt["attempt_id"], sequence=2) if missing: packet.update(record_id=None, sha256=None) row["packets"].append(packet) self.reseal() def test_valid_compact_and_pretty_json_preserve_decoded_text(self): for pretty in (False, True): decoded = self.validate(self.render(pretty=pretty)) self.assertEqual(decoded, self.values) self.assertIn("Café", decoded["CURRENT_ARTIFACT_JSON"]["text"]) self.assertIn("\r\n", decoded["CURRENT_ARTIFACT_JSON"]["text"]) def test_mixed_cohort_uses_worker_requests_and_separate_copy_kinds(self): self.assertFalse(any(a["role"] == "planner" for a in self.sealer.source["runs"][1]["attempts"])) self.set_b1_packet(self.render(pretty=True)) self.reseal() context = self.f.prepare() copies = [c for c in context["copies"] if c["kind"] != "public_packet"] self.assertEqual({c["kind"] for c in copies}, {"planner_input", "b1_continuation_input"}) b1_copy = next(c for c in copies if c["kind"] == "b1_continuation_input") self.assertEqual((self.f.output/b1_copy["path"]).read_bytes(), self.render(pretty=True)) report = self.f.summarize() self.assertFalse(report["semantic_absence_of_restricted_information_certified"]) for series in report["series"].values(): self.assertEqual(series["overall"]["assigned_run_denominator"], 2) for condition in ("C", "B1"): self.assertEqual(series["by_condition"][condition]["counts"]["unassessed"], 1) def test_altered_wrappers_extra_blocks_and_surrounding_prose_rejected(self): valid = self.render() for invalid in (b"Extra instruction\n"+valid, valid+b"Additional text", valid.replace(b"Review the current", b"Score the current", 1), valid+b"\nPERSISTENT_MEMORY_JSON\n[]\n", valid.replace(b"CURRENT_ARTIFACT_JSON\n", b"CURRENT_CRITIQUE_JSON\n", 1)): with self.subTest(invalid=invalid[:30]): with self.assertRaises(ValueError): self.validate(invalid) def test_immutable_packet_task_and_template_hash_mismatches_rejected(self): values = copy.deepcopy(self.values) values["TASK_PACKET_JSON"] += "\nArtificial changed packet." with self.assertRaisesRegex(ValueError, "public packet hash"): self.validate(self.render(values)) with self.assertRaises(ValueError): b1.validate_input(self.render(), self.bounds, "wrong-task", self.public_hash) from unittest import mock with mock.patch.object(b1, "TEMPLATE_SHA256", "0"*64): with self.assertRaisesRegex(ValueError, "template hash"): self.validate(self.render()) def test_archived_hash_and_embedded_public_hash_fail_before_publication(self): self.f.manifest["runs"][1]["packets"][0]["sha256"] = "0"*64 with self.assertRaisesRegex(ValueError, "packet hash"): self.f.prepare() self.assertFalse(self.f.output.exists()) values = copy.deepcopy(self.values) values["TASK_PACKET_JSON"] += "\nArtificial modified source." self.set_b1_packet(self.render(values)) self.reseal() with self.assertRaisesRegex(ValueError, "public packet hash"): self.f.prepare() self.assertFalse(self.f.output.exists()) def test_missing_continuation_request_designation_or_packet_slot_rejected(self): for target in ("request", "packet", "initial"): original = copy.deepcopy(self.f.manifest) with self.subTest(target=target): row = self.f.manifest["runs"][1] if target == "request": row["request_slots"].pop() elif target == "initial": row["request_slots"].pop(0) else: row["packets"] = [] with self.assertRaises(ValueError): self.f.prepare() self.assertFalse(self.f.output.exists()) self.f.manifest = original def test_missing_packet_remains_unknown_with_all_assigned_runs_retained(self): self.f.manifest["runs"][1]["packets"][0].update(record_id=None, sha256=None) self.f.prepare() form = self.f.completed(0) report = self.f.summarize([self.f.write_form(form)]) result = report["series"]["independent:artificial-reviewer-a"] self.assertEqual(result["overall"]["assigned_run_denominator"], 2) self.assertEqual(result["by_condition"]["C"]["counts"]["compliant"], 1) self.assertEqual(result["by_condition"]["B1"]["counts"]["unassessed"], 1) self.assertEqual(result["by_condition"]["B1"]["missing_packet_count"], 1) def test_failed_identical_retry_retains_coverage_and_run_denominator(self): self.add_retry() self.f.prepare() form = self.f.completed(0) form["ratings"][1]["reviewed_packet_ids"].append("b1-retry-packet") report = self.f.summarize([self.f.write_form(form)]) result = report["series"]["independent:artificial-reviewer-a"] self.assertEqual(result["overall"]["assigned_run_denominator"], 2) self.assertEqual(result["by_condition"]["B1"]["counts"]["compliant"], 1) self.assertEqual(result["runs"][1]["declared_packet_count"], 2) def test_missing_retry_or_omitted_retry_does_not_become_free_coverage(self): self.add_retry(missing=True) original = copy.deepcopy(self.f.manifest) self.f.manifest["runs"][1]["request_slots"].pop() self.f.manifest["runs"][1]["packets"].pop() with self.assertRaises(ValueError): self.f.prepare() self.f.manifest = original self.f.prepare() form = self.f.completed(0) result = self.f.summarize([self.f.write_form(form)])["series"]["independent:artificial-reviewer-a"]["by_condition"]["B1"] self.assertEqual(result["counts"]["unassessed"], 1) self.assertEqual(result["missing_packet_count"], 1) def test_retry_cannot_be_reclassified_initial_to_hide_it(self): self.add_retry() self.f.manifest["runs"][1]["request_slots"][-1]["kind"] = "initial" self.f.manifest["runs"][1]["packets"].pop() with self.assertRaisesRegex(ValueError, "retry cannot change"): self.f.prepare() def test_initial_retries_are_explicitly_accounted_for_without_continuation_packets(self): run = copy.deepcopy(self.sealer.source["runs"][1]) slots = copy.deepcopy(self.f.manifest["runs"][1]["request_slots"]) initial_retry = dict(run["attempts"][0], attempt_id="artificial-initial-retry", retry_of="fixture-request-b", status="failed") run["attempts"].append(initial_retry) with self.assertRaisesRegex(ValueError, "every archived adaptation worker"): b1.continuation_attempts(run, slots) slots.append(dict(attempt_id="artificial-initial-retry", kind="initial")) self.assertEqual(set(b1.continuation_attempts(run, slots)), {"artificial-b1-continuation"}) slots[1]["kind"] = "initial" with self.assertRaisesRegex(ValueError, "multiple initial roots"): b1.continuation_attempts(run, slots) def test_reversed_b1_packet_chronology_rejected(self): self.add_retry() self.f.manifest["runs"][1]["packets"].reverse() with self.assertRaisesRegex(ValueError, "chronology"): self.f.prepare() def test_histories_bounds_identity_collisions_and_extra_fields_rejected(self): mutations = [] values = copy.deepcopy(self.values) values["EARLIER_ARTIFACTS_JSON"].append(dict(artifact_id="a0", text="Artificial extra history")) mutations.append(values) values = copy.deepcopy(self.values) values["PREVIOUS_INSTRUCTIONS_JSON"] = [dict(instruction_id="i"+str(i), text="Artificial directive") for i in range(13)] mutations.append(values) for field, value in (("artifact_id", "R1"), ("artifact_id", "a1"), ("text", 1), ("memory", [])): values = copy.deepcopy(self.values) values["CURRENT_ARTIFACT_JSON"][field] = value mutations.append(values) values = copy.deepcopy(self.values) values["PREVIOUS_INSTRUCTIONS_JSON"][0]["instruction_id"] = "a3" mutations.append(values) values = copy.deepcopy(self.values) values["PREVIOUS_INSTRUCTIONS_JSON"][0]["score"] = 1 mutations.append(values) for values in mutations: with self.assertRaises(ValueError): self.validate(self.render(values)) with self.assertRaisesRegex(ValueError, "byte bound"): b1.validate_input(self.render(), dict(max_input_bytes=1), self.tid, self.public_hash) def test_empty_history_is_valid_but_memory_and_critique_are_not_added(self): values = copy.deepcopy(self.values) values["EARLIER_ARTIFACTS_JSON"] = [] values["PREVIOUS_INSTRUCTIONS_JSON"] = [] self.assertEqual(self.validate(self.render(values)), values) self.assertEqual(set(self.validate(self.render(values))), set(b1.PLACEHOLDERS)) def test_utf8_duplicate_json_keys_unresolved_markers_and_non_json_rejected(self): valid = self.render() for invalid in (valid+b"\xff", valid.replace(b'"artifact_id": "a3"', b'"artifact_id":"a3","artifact_id":"a4"'), valid.replace(b'"artifact_id": "a3"', b'"artifact_id": "{{UNKNOWN}}"'), valid.replace(b'"artifact_id": "a3"', b'"artifact_id": NaN')): with self.assertRaises(ValueError): self.validate(invalid) def test_human_verdicts_separate_conditions_and_independent_series(self): self.f.prepare() form = self.f.completed(1) form["ratings"][0].update(violation=0, findings=[]) report = self.f.summarize([self.f.write_form(form)]) result = report["series"]["independent:artificial-reviewer-a"] self.assertEqual(result["by_condition"]["C"]["counts"], dict(compliant=1, noncompliant=0, unassessed=0)) self.assertEqual(result["by_condition"]["B1"]["counts"], dict(compliant=0, noncompliant=1, unassessed=0)) self.assertEqual(report["series"]["adjudicated"]["overall"]["counts"]["unassessed"], 2) self.assertEqual(report["series"]["independent:artificial-reviewer-b"]["overall"]["counts"]["unassessed"], 2) self.assertFalse(report["semantic_absence_of_restricted_information_certified"]) def test_unknown_contract_request_kind_duplicate_slot_and_wrong_role_rejected(self): original = copy.deepcopy(self.f.manifest) for case in ("contract", "version", "kind", "duplicate", "role", "extra"): with self.subTest(case=case): row = self.f.manifest["runs"][1] if case == "contract": row["contract"] = "unsupported/1.0" elif case == "version": row["contract"] = "b1-generic-continuation/2.0" elif case == "kind": row["request_slots"][0]["kind"] = "scoring" elif case == "duplicate": row["request_slots"].append(copy.deepcopy(row["request_slots"][0])) elif case == "role": row["request_slots"][0]["attempt_id"] = "artificial-planner-attempt-0" else: row["request_slots"][0]["score"] = 0 with self.assertRaises(ValueError): self.f.prepare() self.f.manifest = copy.deepcopy(original) def test_unsupported_contract_with_its_own_fields_is_explicitly_rejected(self): self.f.manifest["runs"][1]["contract"] = "b1-generic-continuation/2.0" with self.assertRaisesRegex(ValueError, "unsupported permitted-input contract"): self.f.prepare() if __name__ == "__main__": unittest.main()