#!/usr/bin/env python3 """Offline, explicit-input assembler for the matched treatment planner template. Development tooling. Structural checks are not semantic provenance validation. """ import argparse import hashlib import json import os from pathlib import Path import re import sys NOTICE = "Development observation-assembly tooling; no semantic provenance validation or model execution." TEMPLATE_SHA256 = "29cde9654be46b7c9a883cebe94ddd707c4ababa875fa2a6c62e1de7a47de209" PLACEHOLDERS = {"TASK_PACKET_JSON", "CURRENT_ARTIFACT_JSON", "EARLIER_ARTIFACTS_JSON", "PREVIOUS_INSTRUCTIONS_JSON", "PERSISTENT_MEMORY_JSON"} MARKER = re.compile(r"\{\{[^{}\r\n]*\}\}") BOUND_FIELDS = {"max_earlier_artifacts", "max_previous_instructions", "max_memory_entries", "max_memory_tokens", "max_task_bytes", "max_artifact_bytes", "max_instruction_bytes", "max_memory_bytes", "max_rendered_bytes"} EXCLUDED_BASENAMES = {"rating-form.json", "adjudication-form.json", "coordinator-manifest.json", "packet-key.json", "archive-inventory.json"} class AssemblyError(ValueError): pass def require(test, message): if not test: raise AssemblyError(message) def keys(value, expected, label): require(isinstance(value, dict) and set(value) == set(expected), f"{label}: missing or excluded/unknown fields") def text(value, label): require(isinstance(value, str) and bool(value.strip()), f"{label}: expected nonempty string") value.encode("utf-8") return value def integer(value, minimum, label): require(type(value) is int and value >= minimum, f"{label}: expected integer >= {minimum}") return value def sha(content): return hashlib.sha256(content).hexdigest() def canonical(value): return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False) def parse_json(content, label): def unique(pairs): result = {} for key, value in pairs: require(key not in result, f"{label}: duplicate JSON key {key}") result[key] = value return result def invalid(value): raise AssemblyError(f"{label}: invalid numeric constant {value}") return json.loads(content.decode("utf-8"), object_pairs_hook=unique, parse_constant=invalid) def new_output(path): path = Path(path).absolute() require(not os.path.lexists(path), f"refusing existing output destination: {path}") require(path.parent.is_dir(), "output parent must already exist") return path.resolve() def read_declared(descriptor, parent, limit, label, provenance): relative = text(descriptor["path"], label+" path") original = parent/relative path = original.resolve() # Catch recognizable accidental assessment/operational paths. This is not # a classifier: copied/renamed restricted content can still be mislabeled. for candidate in (original, path): parts = {p.casefold() for p in candidate.parts} require(not ({"assessor-only", "resource-accounting"} & parts) and candidate.name.casefold() not in EXCLUDED_BASENAMES, f"{label}: recognizable excluded assessment/operational path") expected = descriptor["sha256"] require(isinstance(expected, str) and re.fullmatch(r"[0-9a-f]{64}", expected) is not None, label+": expected SHA-256") require(path.is_file(), label+": missing/non-file declared input") with path.open("rb") as handle: content = handle.read(limit+1) require(len(content) <= limit, label+": byte bound exceeded; no truncation is emitted") require(sha(content) == expected, label+": source hash mismatch") decoded = content.decode("utf-8") # No universal-newline conversion. provenance.append(dict(channel=label, source_path=str(path), sha256=expected, bytes=len(content))) return decoded def task_identifiers(packet, task_id): require(re.search(r"^\ufeff?#\s+"+re.escape(task_id)+r"(?:\s|:|$)", packet, re.MULTILINE) is not None, "task_id does not match public packet heading") requirements = re.findall(r"^- \*\*(R[1-9][0-9]*):\*\*", packet, re.MULTILINE) sources = re.findall(r"^### (S[1-9][0-9]*)(?:\s|$)", packet, re.MULTILINE) require(requirements and sources, "public packet must define requirement bullets and authorized source headings") require(len(requirements) == len(set(requirements)) and len(sources) == len(set(sources)), "duplicate task/source identity") return set(requirements), set(sources) def validate_evidence(evidence, catalogs, allow_empty=False, max_references=None): """Validate reference shape/membership only, not location truth or support.""" require(isinstance(evidence, list) and (allow_empty or evidence), "memory requires evidence references") if max_references is not None: require(len(evidence) <= max_references, "evidence reference bound exceeded") for reference in evidence: require(isinstance(reference, dict), "evidence must be an object") identifiers = set(reference) & set(catalogs) require(len(identifiers) == 1 and set(reference) == identifiers | {"location"}, "evidence requires exactly one identifier and location") kind = next(iter(identifiers)) require(isinstance(reference[kind], str) and reference[kind] in catalogs[kind], "evidence identifier is not supplied in this observation") text(reference["location"], "evidence location") def validate_typed_memory(memory, catalogs, used_ids, max_entries, max_references=None): """Reuse the matched memory interface; mutate only the supplied identity set.""" require(isinstance(memory, list) and len(memory) <= max_entries, "excess/invalid memory entries") for entry in memory: keys(entry, ("id", "kind", "statement", "evidence", "status"), "memory entry") text(entry["id"], "memory id") require(entry["id"] not in used_ids, "duplicate/ambiguous input identity: "+entry["id"]) used_ids.add(entry["id"]) require(entry["kind"] in ("observation", "provisional_tactic"), "invalid memory kind") require(entry["status"] in ("current", "unresolved", "superseded"), "invalid memory status") text(entry["statement"], "memory statement") validate_evidence(entry["evidence"], catalogs, max_references=max_references) def assemble(manifest_path, model_dir, provenance_dir): manifest_path = Path(manifest_path).resolve() model_dir, provenance_dir = new_output(model_dir), new_output(provenance_dir) require(model_dir != provenance_dir and model_dir not in provenance_dir.parents and provenance_dir not in model_dir.parents, "model and provenance destinations must be separate, disjoint directories") manifest_bytes = manifest_path.read_bytes() manifest = parse_json(manifest_bytes, "observation manifest") keys(manifest, ("manifest_version", "assembly_id", "template", "task_packet", "current_artifact", "earlier_artifacts", "previous_instructions", "memory", "bounds"), "manifest") require(manifest["manifest_version"] == "1.0", "unsupported manifest version") text(manifest["assembly_id"], "assembly_id") bounds = manifest["bounds"] keys(bounds, BOUND_FIELDS, "bounds") for key, value in bounds.items(): integer(value, 0 if key in ("max_earlier_artifacts", "max_previous_instructions", "max_memory_entries") else 1, key) require(bounds["max_earlier_artifacts"] <= 2, "reference permits at most two earlier artifacts") require(bounds["max_previous_instructions"] <= 12, "matched history policy permits at most twelve directives") require(bounds["max_memory_tokens"] <= 1200, "reference memory cap cannot exceed 1,200 tokens") earlier = manifest["earlier_artifacts"] instructions = manifest["previous_instructions"] require(isinstance(earlier, list) and len(earlier) <= bounds["max_earlier_artifacts"], "excess/invalid earlier artifact history") require(isinstance(instructions, list) and len(instructions) <= bounds["max_previous_instructions"], "excess/invalid instruction history") provenance = [] parent = manifest_path.parent keys(manifest["template"], ("path", "sha256"), "template") require(manifest["template"]["sha256"] == TEMPLATE_SHA256, "must use the unchanged matched treatment reference") template = read_declared(manifest["template"], parent, 65536, "template", provenance) expected_markers = {"{{"+name+"}}" for name in PLACEHOLDERS} require(set(MARKER.findall(template)) == expected_markers and all(template.count(marker) == 1 for marker in expected_markers), "unexpected or unresolved template placeholders") task = manifest["task_packet"] keys(task, ("task_id", "path", "sha256"), "task_packet") text(task["task_id"], "task_id") packet = read_declared(task, parent, bounds["max_task_bytes"], "public_task_packet", provenance) requirement_ids, source_ids = task_identifiers(packet, task["task_id"]) require(task["task_id"] not in requirement_ids | source_ids, "ambiguous task/requirement/source identity") used = {task["task_id"], *requirement_ids, *source_ids} def unique_id(value, label): text(value, label) require(value not in used, "duplicate/ambiguous input identity: "+value) used.add(value) artifact_ids, artifact_paths = set(), set() def artifact(descriptor, label): keys(descriptor, ("artifact_id", "sequence", "path", "sha256"), label) unique_id(descriptor["artifact_id"], "artifact_id") integer(descriptor["sequence"], 1, "artifact sequence") path = (parent/text(descriptor["path"], "artifact path")).resolve() require(path not in artifact_paths, "duplicate current/earlier artifact source path") artifact_paths.add(path) artifact_ids.add(descriptor["artifact_id"]) return dict(artifact_id=descriptor["artifact_id"], text=read_declared(descriptor, parent, bounds["max_artifact_bytes"], label, provenance)) current_descriptor = manifest["current_artifact"] current = artifact(current_descriptor, "current_artifact") older, last_sequence = [], 0 for descriptor in earlier: value = artifact(descriptor, "earlier_artifact") require(last_sequence < descriptor["sequence"] < current_descriptor["sequence"], "earlier artifacts must be strictly chronological and before current") last_sequence = descriptor["sequence"] older.append(value) previous, last_sequence = [], 0 for descriptor in instructions: keys(descriptor, ("instruction_id", "sequence", "path", "sha256"), "previous instruction") unique_id(descriptor["instruction_id"], "instruction_id") sequence = integer(descriptor["sequence"], 1, "instruction sequence") require(last_sequence < sequence <= current_descriptor["sequence"], "instructions must be chronological and already submitted by current turn") last_sequence = sequence directive = read_declared(descriptor, parent, bounds["max_instruction_bytes"], "worker_instruction", provenance) text(directive, "submitted worker directive") previous.append(dict(instruction_id=descriptor["instruction_id"], text=directive)) memory_descriptor = manifest["memory"] keys(memory_descriptor, ("path", "sha256", "token_attestation"), "memory input") memory_text = read_declared(memory_descriptor, parent, bounds["max_memory_bytes"], "typed_memory", provenance) memory = parse_json(memory_text.encode("utf-8"), "typed memory") catalogs = dict(artifact_id=artifact_ids, requirement_id=requirement_ids, source_id=source_ids) validate_typed_memory(memory, catalogs, used, bounds["max_memory_entries"]) memory_value = canonical(memory) require(len(memory_value.encode("utf-8")) <= bounds["max_memory_bytes"], "serialized memory byte bound exceeded") attestation = memory_descriptor["token_attestation"] if attestation is None: require(not memory, "nonempty memory requires a caller token-count attestation") else: keys(attestation, ("tokenizer", "serialized_sha256", "token_count"), "memory token attestation") text(attestation["tokenizer"], "declared tokenizer") count = integer(attestation["token_count"], 1, "declared memory token count") require(count <= bounds["max_memory_tokens"], "declared memory token cap exceeded") require(attestation["serialized_sha256"] == sha(memory_value.encode("utf-8")), "token attestation is not bound to serialized memory") values = dict(TASK_PACKET_JSON=packet, CURRENT_ARTIFACT_JSON=current, EARLIER_ARTIFACTS_JSON=older, PREVIOUS_INSTRUCTIONS_JSON=previous, PERSISTENT_MEMORY_JSON=memory) encoded = {"{{"+key+"}}": canonical(value) for key, value in values.items()} rendered = MARKER.sub(lambda match: encoded[match.group(0)], template) require(MARKER.search(rendered) is None, "unresolved placeholder in rendered prompt or supplied text") rendered_bytes = rendered.encode("utf-8") require(len(rendered_bytes) <= bounds["max_rendered_bytes"], "rendered prompt byte bound exceeded; no truncation") record = dict(assembler_version="1.0", notice=NOTICE, defaults_unfrozen=True, assembly_id=manifest["assembly_id"], manifest_path=str(manifest_path), manifest_sha256=sha(manifest_bytes), sources=provenance, bounds=bounds, model_input_sha256=sha(rendered_bytes), model_input_bytes=len(rendered_bytes), chronology=dict(current_artifact_sequence=current_descriptor["sequence"], earlier_artifact_sequences=[r["sequence"] for r in earlier], instruction_sequences=[r["sequence"] for r in instructions]), memory_token_attestation=attestation, token_count_independently_verified=False, semantic_provenance_validated=False, assembly_complete=True) model_dir.mkdir() provenance_dir.mkdir() with (model_dir/"planner-input.txt").open("xb") as handle: handle.write(rendered_bytes) with (provenance_dir/"assembly-record.json").open("xb") as handle: handle.write((json.dumps(record, indent=2, ensure_ascii=False, allow_nan=False)+"\n").encode("utf-8")) return dict(assembler_version="1.0", notice=NOTICE, model_input=str(model_dir/"planner-input.txt"), provenance_record=str(provenance_dir/"assembly-record.json"), model_input_sha256=sha(rendered_bytes)) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--model-dir", type=Path, required=True) parser.add_argument("--provenance-dir", type=Path, required=True) args = parser.parse_args(argv) try: result = assemble(args.manifest, args.model_dir, args.provenance_dir) except (AssemblyError, OSError, KeyError, TypeError, ValueError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2)) return 0 if __name__ == "__main__": sys.exit(main())