"""Temporary artificial handoff fixtures, not planner runs or study observations.""" import copy import json from pathlib import Path import subprocess import sys import unittest import assemble_observation as observation import validate_planner_response as validator import test_assemble_observation as assembly_fixtures class ResponseValidationTests(unittest.TestCase): def setUp(self): # Reuse the existing artificial assembly fixture, never real task outputs. self.fixture = assembly_fixtures.ObservationTests() self.fixture.setUp() self.addCleanup(self.fixture.doCleanups) self.root = self.fixture.root self.fixture.assemble() self.input = self.fixture.model/"planner-input.txt" self.raw = self.root/"artificial-raw-response.json" self.manifest_path = self.root/"validation-manifest.json" self.staging = self.root/"new-proposals" self.audit = self.root/"new-validation-record" self.response = dict(observed_gap="Artificial structural fixture; not a study observation.", evidence=[dict(artifact_id="a3", location="line 1")], uncertainty="Artificial unverified fixture.", next_prompt=" \tArtificial directive for a formatting check. Café.\r\n", memory=[dict(id="m1", kind="observation", statement="Artificial note only.", evidence=[dict(source_id="S1", location="L1")], status="unresolved")]) base = Path(validator.__file__).resolve().parent self.manifest = json.loads((base/"response-validation.template.json").read_text()) self.manifest.update(validation_id="PRIVATE-VALIDATION-ID", assembled_input=dict(path=str(self.input), sha256=observation.sha(self.input.read_bytes())), planner_response=dict(path=str(self.raw), sha256=None)) self.attest() def attest(self): for field, value in (("next_prompt", self.response["next_prompt"].encode("utf-8")), ("memory", observation.canonical(self.response["memory"]).encode("utf-8"))): self.manifest["token_attestations"][field] = dict(tokenizer="artificial-count-fixture-not-a-tokenizer", sha256=observation.sha(value), token_count=100) def prepare(self, raw=None): content = json.dumps(self.response, ensure_ascii=False).encode("utf-8") if raw is None else raw self.raw.write_bytes(content) self.manifest["planner_response"]["sha256"] = observation.sha(content) self.manifest_path.write_bytes(json.dumps(self.manifest).encode()) def validate(self, raw=None): self.prepare(raw) return validator.validate_response(self.manifest_path, self.staging, self.audit) def assert_no_staging(self): self.assertFalse(self.staging.exists()) self.assertFalse(self.audit.exists()) def test_exact_decoded_directive_canonical_memory_and_hash_binding(self): before_input = self.input.read_bytes() self.prepare() before_response = self.raw.read_bytes() result = validator.validate_response(self.manifest_path, self.staging, self.audit) self.assertEqual((self.staging/"next-prompt.txt").read_bytes(), self.response["next_prompt"].encode()) self.assertEqual((self.staging/"proposed-memory.json").read_bytes(), observation.canonical(self.response["memory"]).encode()) self.assertEqual(self.input.read_bytes(), before_input) self.assertEqual(self.raw.read_bytes(), before_response) record = json.loads((self.audit/"validation-record.json").read_text()) self.assertEqual(record["assembled_input_sha256"], observation.sha(before_input)) self.assertEqual(record["raw_response_sha256"], observation.sha(before_response)) self.assertFalse(record["tokenizer_counts_independently_verified"]) self.assertFalse(record["semantic_support_verified"]) self.assertFalse(record["restricted_requests_verified"]) self.assertFalse(result["directive_delivered"]) self.assertFalse(result["live_memory_replaced"]) self.assertEqual({p.name for p in self.staging.iterdir()}, {"next-prompt.txt", "proposed-memory.json"}) self.assertNotIn(b"PRIVATE-VALIDATION-ID", (self.staging/"next-prompt.txt").read_bytes()) def test_only_five_fields_and_correct_types(self): original = copy.deepcopy(self.response) for field, value in [("observed_gap", None), ("uncertainty", []), ("evidence", "not an array"), ("memory", {"append": []}), ("next_prompt", 1), ("next_prompt", " \r\n\t")]: self.response = copy.deepcopy(original) self.response[field] = value with self.subTest(field=field, value=value), self.assertRaises(observation.AssemblyError): self.validate() self.assert_no_staging() self.response = copy.deepcopy(original) self.response["score"] = 1 with self.assertRaisesRegex(observation.AssemblyError, "excluded/unknown"): self.validate() del self.response["score"] del self.response["observed_gap"] with self.assertRaisesRegex(observation.AssemblyError, "excluded/unknown"): self.validate() def test_surrounding_prose_code_fences_and_duplicate_keys_rejected(self): raw = observation.canonical(self.response).encode() cases = [b"Explanation first\n"+raw, raw+b"\nFurther explanation", b"```json\n"+raw+b"\n```", b'{"observed_gap":"duplicate",'+raw[1:]] for content in cases: with self.subTest(content=content[:30]), self.assertRaises((observation.AssemblyError, ValueError)): self.validate(content) self.assert_no_staging() def test_invalid_evidence_ids_locations_and_extra_reference_fields(self): for reference in [dict(artifact_id="a99", location="line 1"), dict(requirement_id="R99", location="requirement"), dict(source_id="S99", location="L1"), dict(artifact_id="a3", location=""), dict(artifact_id="a3", source_id="S1", location="line 1")]: self.response["evidence"] = [reference] with self.subTest(reference=reference), self.assertRaises(observation.AssemblyError): self.validate() self.assert_no_staging() def test_memory_schema_duplicate_ids_and_unsupported_references(self): original = copy.deepcopy(self.response["memory"]) for mutation in [lambda m: m[0].pop("status"), lambda m: m[0].update(kind="requirement"), lambda m: m[0].update(status="verified_success"), lambda m: m[0].update(id="a3"), lambda m: m[0].update(evidence=[dict(artifact_id="missing", location="line 1")]), lambda m: m.append(copy.deepcopy(m[0]))]: self.response["memory"] = copy.deepcopy(original) mutation(self.response["memory"]) with self.assertRaises(observation.AssemblyError): self.validate() self.assert_no_staging() def test_full_replacement_can_reuse_old_ids_or_be_empty_without_live_update(self): self.fixture.set_memory() self.fixture.assemble(self.root/"input-with-memory", self.root/"input-with-memory-audit") self.input = self.root/"input-with-memory"/"planner-input.txt" self.manifest["assembled_input"] = dict(path=str(self.input), sha256=observation.sha(self.input.read_bytes())) old_memory_bytes = (self.root/"memory.json").read_bytes() self.validate() # m1 is retained as an ID, not rejected as a duplicate old ID. self.assertEqual((self.root/"memory.json").read_bytes(), old_memory_bytes) self.staging, self.audit = self.root/"empty-proposal", self.root/"empty-audit" self.response["memory"] = [] self.response["observed_gap"] = "" self.response["uncertainty"] = "" self.response["evidence"] = [] self.attest() self.validate() self.assertEqual((self.staging/"proposed-memory.json").read_bytes(), b"[]") self.assertEqual((self.root/"memory.json").read_bytes(), old_memory_bytes) def test_wrong_input_or_raw_response_hash_rejected(self): self.manifest["assembled_input"]["sha256"] = "0"*64 with self.assertRaisesRegex(observation.AssemblyError, "assembled input hash mismatch"): self.validate() self.assert_no_staging() self.manifest["assembled_input"]["sha256"] = observation.sha(self.input.read_bytes()) self.prepare() self.manifest["planner_response"]["sha256"] = "0"*64 self.manifest_path.write_bytes(json.dumps(self.manifest).encode()) with self.assertRaisesRegex(observation.AssemblyError, "planner response hash mismatch"): validator.validate_response(self.manifest_path, self.staging, self.audit) def test_same_response_is_checked_against_supplied_context_not_an_external_catalog(self): self.fixture.manifest["current_artifact"]["artifact_id"] = "different-current" self.fixture.assemble(self.root/"different-input", self.root/"different-input-audit") path = self.root/"different-input"/"planner-input.txt" self.manifest["assembled_input"] = dict(path=str(path), sha256=observation.sha(path.read_bytes())) with self.assertRaisesRegex(observation.AssemblyError, "identifier is not supplied"): self.validate() self.assert_no_staging() def test_modified_input_wrapper_and_noncanonical_blocks_are_rejected(self): path = self.root/"altered-input.txt" modified = self.input.read_bytes().replace(b"Principles:", b"Altered policy:") path.write_bytes(modified) self.manifest["assembled_input"] = dict(path=str(path), sha256=observation.sha(modified)) with self.assertRaisesRegex(observation.AssemblyError, "unchanged matched treatment wrapper"): self.validate() modified = self.input.read_bytes().replace(b'"artifact_id":"a3"', b'"artifact_id": "a3"') path.write_bytes(modified) self.manifest["assembled_input"]["sha256"] = observation.sha(modified) with self.assertRaisesRegex(observation.AssemblyError, "canonical"): self.validate() def test_attestations_bound_to_exact_values_and_caps(self): for field, bad in [("next_prompt", 801), ("memory", 1201), ("next_prompt", True), ("memory", 1.0)]: self.attest() self.manifest["token_attestations"][field]["token_count"] = bad with self.subTest(field=field, value=bad), self.assertRaises(observation.AssemblyError): self.validate() for field in ("next_prompt", "memory"): self.attest() self.manifest["token_attestations"][field]["sha256"] = "0"*64 with self.subTest(field=field), self.assertRaisesRegex(observation.AssemblyError, "attestation hash mismatch"): self.validate() self.assert_no_staging() def test_byte_entry_and_reference_bounds_reject_without_truncation(self): for field, value in [("max_response_bytes", 1), ("max_next_prompt_bytes", 1), ("max_memory_bytes", 1), ("max_memory_entries", 0), ("max_evidence_references", 0), ("max_next_prompt_tokens", 801)]: old = self.manifest["bounds"][field] self.manifest["bounds"][field] = value with self.subTest(field=field), self.assertRaises(observation.AssemblyError): self.validate() self.manifest["bounds"][field] = old self.assert_no_staging() def test_original_assembly_sources_are_not_needed_or_read(self): descriptors = [self.fixture.manifest[k] for k in ("template", "task_packet", "current_artifact", "memory")] descriptors += self.fixture.manifest["earlier_artifacts"]+self.fixture.manifest["previous_instructions"] for path in {(self.root/d["path"]).resolve() for d in descriptors}: path.unlink() # Only temporary fixture-owned sources, not repository assets. self.validate() self.assertTrue((self.audit/"validation-record.json").is_file()) def test_existing_destinations_refused_and_cli_success_is_staging_only(self): self.prepare() command = [sys.executable, "-B", str(Path(validator.__file__).resolve()), "--manifest", str(self.manifest_path), "--staging-dir", str(self.staging), "--validation-dir", str(self.audit)] run = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(run.returncode, 0, run.stderr) self.assertFalse(json.loads(run.stdout)["directive_delivered"]) before = (self.staging/"proposed-memory.json").read_bytes() repeated = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(repeated.returncode, 2) self.assertEqual(repeated.stdout, "") self.assertIn("refusing existing output", json.loads(repeated.stderr)["error"]) self.assertEqual((self.staging/"proposed-memory.json").read_bytes(), before) if __name__ == "__main__": unittest.main()