#!/usr/bin/env python3 """Offline, coordinator-only snapshot/sealing of explicitly inventoried runs. Development tooling; no endpoint inference, grading, models, or experiments. """ import argparse import copy from datetime import datetime, timezone from decimal import Decimal import hashlib import json import os from pathlib import Path import sys import tempfile import uuid BASE = Path(__file__).resolve().parent KIT = BASE.parent/"development-tasks"/"assessor-only"/"manual-assessment" sys.path.insert(0, str(KIT)) sys.path.insert(0, str(BASE.parent/"resource-accounting")) from score_forms import ValidationError, array, digest, keys, manifest_data, registry_data, require, strings, text, utc import account_resources NOTICE = "Development run-sealing tooling; hashes and sealing do not prove preregistration, completeness, or prior immutability." KINDS = {"artifact": "artifacts", "submitted_prompt": "submitted_prompts", "memory_snapshot": "memory_snapshots", "configuration": "configuration", "usage_ledger": "usage_records", "usage_raw": "usage_records", "operational_record": "operational_records"} CATEGORIES = set(KINDS.values()) RUN_STATUSES = ("completed", "incomplete", "failed", "cancelled") ATTEMPT_STATUSES = (*account_resources.STATUSES, "incomplete") def sha(content): return hashlib.sha256(content).hexdigest() def parse_json(content, label, decimal=False): def unique(pairs): result = {} for key, value in pairs: require(key not in result, f"{label}: duplicate JSON key {key}") result[key] = value return result def invalid(value): raise ValidationError(f"{label}: invalid numeric constant {value}") return json.loads(content.decode("utf-8"), object_pairs_hook=unique, parse_float=Decimal if decimal else float, parse_constant=invalid) def json_bytes(value): return (json.dumps(value, indent=2, allow_nan=False)+"\n").encode("utf-8") def read_source(path, expected=None): require(path.is_file(), f"missing/non-file declared source: {path}") content = path.read_bytes() if expected is not None: digest(expected, "expected sha256") require(sha(content) == expected, f"source hash mismatch: {path}") return content def write_bytes(path, content): path.parent.mkdir(parents=True, exist_ok=True) with path.open("xb") as handle: handle.write(content) handle.flush() os.fsync(handle.fileno()) def utc_optional(value, label): return None if value is None else utc(value, label) def validate_inventory(source, records, tasks): keys(source, ("input_inventory_version", "batch_id", "collection_completed", "independent_rater_ids", "conditions", "records", "runs"), "input inventory") require(source["input_inventory_version"] == "1.0", "unsupported input inventory version") require(source["collection_completed"] is True, "coordinator must declare collection completed") contract = dict(manifest_version="1.0", batch_id=source["batch_id"], seal_record_id="validation-only", independent_rater_ids=source["independent_rater_ids"], conditions=source["conditions"], assignments=[]) _, conditions, _, _ = manifest_data(contract, tasks) runs, cells, attempts, users = {}, set(), {}, {rid: set() for rid in records} now = datetime.now(timezone.utc) for run in array(source["runs"], "runs"): keys(run, ("run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id", "run_status", "ended_at_utc", "endpoint_selection", "record_ids", "record_availability", "attempts"), "run") rid = text(run["run_id"], "run_id") require(rid not in runs, "duplicate run ID") require(run["task_id"] in tasks, "unknown task") task = tasks[run["task_id"]] require(all(run[k] == task[k] for k in ("task_version", "rubric_version")), "incompatible task/rubric version") cid, rep = run["condition_id"], run["repetition_id"] require(cid in conditions and rep in conditions[cid], "undeclared condition/repetition") cell = (cid, run["task_id"], rep) require(cell not in cells, "duplicate condition/task/repetition run cell") cells.add(cell) require(run["run_status"] in RUN_STATUSES, "invalid run status") end = utc_optional(run["ended_at_utc"], "run ended_at_utc") require(end is None or end <= now, "run end cannot be in the future") strings(run["record_ids"], "run record_ids") require(len(run["record_ids"]) == len(set(run["record_ids"])), "duplicate run record reference") for record_id in run["record_ids"]: require(record_id in records, "run references unknown record") users[record_id].add(rid) keys(run["record_availability"], CATEGORIES, "record_availability") for category, declaration in run["record_availability"].items(): keys(declaration, ("state", "note"), "availability declaration") state = declaration["state"] count = sum(KINDS[records[r]["kind"]] == category for r in run["record_ids"]) require(state in ("available", "partial", "unavailable", "not_applicable"), "unknown availability state") require((count > 0) == (state in ("available", "partial")), "availability contradicts supplied records") require(declaration["note"] is None or isinstance(declaration["note"], str), "invalid availability note") if state != "available": text(declaration["note"], "partial/unavailable/not-applicable explanation") endpoint = run["endpoint_selection"] keys(endpoint, ("artifact_id", "basis"), "endpoint_selection") text(endpoint["basis"], "explicit endpoint selection basis") final_id = endpoint["artifact_id"] if final_id is None: require(run["run_status"] != "completed", "completed run requires a designated final artifact") else: require(final_id in run["record_ids"] and records[final_id]["kind"] == "artifact", "endpoint must designate this run's artifact record") run_attempts = array(run["attempts"], "attempts") require(run_attempts, "declare attempts explicitly, including incomplete attempts") for attempt in run_attempts: keys(attempt, ("attempt_id", "phase", "role", "status", "retry_of", "started_at_utc", "ended_at_utc", "record_ids"), "attempt") aid = text(attempt["attempt_id"], "attempt_id") require(aid not in attempts, "duplicate attempt ID") require(attempt["phase"] in account_resources.PHASES and attempt["role"] in account_resources.ROLES, "invalid attempt phase/role") require(attempt["status"] in ATTEMPT_STATUSES, "invalid attempt status") if attempt["retry_of"] is not None: text(attempt["retry_of"], "retry_of") start = utc_optional(attempt["started_at_utc"], "attempt start") finish = utc_optional(attempt["ended_at_utc"], "attempt end") require(start is None or start <= now, "attempt start cannot be in the future") require(finish is None or finish <= now, "attempt end cannot be in the future") require(start is None or finish is None or start <= finish, "attempt ends before it starts") require(end is None or finish is None or finish <= end, "attempt ends after run end") strings(attempt["record_ids"], "attempt record_ids") require(len(attempt["record_ids"]) == len(set(attempt["record_ids"])) and set(attempt["record_ids"]) <= set(run["record_ids"]), "invalid/duplicate attempt record reference") attempts[aid] = dict(run_id=rid, **attempt) runs[rid] = run require(runs, "declare at least one run") require(any(r["endpoint_selection"]["artifact_id"] is not None for r in runs.values()), "at least one designated deliverable is required to emit a compatible packet manifest; no run is silently dropped") for record_id, referenced_by in users.items(): require(referenced_by, "declared file is not assigned to any run") if records[record_id]["kind"] == "artifact": require(len(referenced_by) == 1, "artifact records cannot be shared across run identities") for aid, attempt in attempts.items(): seen, parent = {aid}, attempt["retry_of"] while parent is not None: require(parent in attempts and parent not in seen, "missing/cyclic attempt retry reference") other = attempts[parent] require(all(other[k] == attempt[k] for k in ("run_id", "phase", "role")), "attempt retry identity mismatch") seen.add(parent) parent = other["retry_of"] return runs, attempts, users def validate_usage(records, snapshots, runs, attempts, users): seen_requests = set() for record_id, record in records.items(): if record["kind"] != "usage_ledger": continue ledger = parse_json(snapshots[record_id], "resource ledger", decimal=True) ledger_runs, requests = account_resources.validate(ledger) require(set(ledger_runs) == users[record_id], "usage-ledger run scope must match referring runs") for rid, row in ledger_runs.items(): require(rid in runs and all(row[k] == runs[rid][k] for k in ("condition_id", "task_id", "repetition_id")), "usage-ledger run identity mismatch") for request in requests: aid = request["request_id"] require(aid not in seen_requests, "duplicate request across declared usage ledgers") require(aid in attempts, "usage request missing from declared attempt inventory") require(all(request[k] == attempts[aid][k] for k in ("run_id", "phase", "role", "retry_of")), "usage request/attempt identity mismatch") seen_requests.add(aid) def verify_payloads(root, inventory, expected_inventory_hash): require(sha((root/"archive-inventory.json").read_bytes()) == expected_inventory_hash, "archive inventory write validation failed") require((root/"archive-inventory.sha256").read_text().strip() == expected_inventory_hash, "inventory digest file mismatch") for row in inventory["records"] + inventory["support_files"]: relative = Path(row["path"]) require(not relative.is_absolute() and ".." not in relative.parts, "unsafe archive relative path") content = (root/relative).read_bytes() require(len(content) == row["bytes"] and sha(content) == row["sha256"], f"archive copy validation failed: {relative}") def publish_manifest(root, content): """Atomic no-clobber publication of the ONLY seal/commit record. The temporary file is a sibling outside the archive. A failed/interrupted write or failed hard link leaves no committed packet manifest in the archive. """ name = None try: with tempfile.NamedTemporaryFile(prefix=".pending-run-seal-", dir=root.parent, delete=False) as handle: name = Path(handle.name) handle.write(content) handle.flush() os.fsync(handle.fileno()) require(name.read_bytes() == content, "seal manifest staging validation failed") os.link(name, root/"packet-build-manifest.json") # Atomic, fails if target exists. finally: if name is not None: try: name.unlink() except OSError: pass # A leftover sibling is not the archive's commit record. def seal_inventory(inventory_path, output_dir, registry_path=None): inventory_path = Path(inventory_path).resolve() registry_path = Path(registry_path or KIT/"task-registry.json").resolve() output = Path(output_dir).absolute() require(not os.path.lexists(output), f"refusing existing archive destination: {output}") require(output.parent.is_dir(), "archive parent must already exist") output = output.resolve() source_bytes = read_source(inventory_path) registry_bytes = read_source(registry_path) source = parse_json(source_bytes, "input inventory") registry = parse_json(registry_bytes, "registry") require(registry["kit_version"] == "1.0", "incompatible registry version") tasks, _, _ = registry_data(registry, registry_path.parent) records, snapshots, source_paths = {}, {}, set() for row in array(source["records"], "records"): keys(row, ("record_id", "kind", "path", "sha256"), "source record") rid = text(row["record_id"], "record_id") require(rid not in records and row["kind"] in KINDS, "duplicate record ID or unknown record kind") path = (inventory_path.parent/text(row["path"], "record path")).resolve() require(path not in source_paths, "duplicate source path; share one record ID explicitly") snapshots[rid] = read_source(path, row["sha256"]) records[rid] = row source_paths.add(path) runs, attempts, users = validate_inventory(source, records, tasks) validate_usage(records, snapshots, runs, attempts, users) for run in runs.values(): endpoint = run["endpoint_selection"]["artifact_id"] if endpoint is not None: snapshots[endpoint].decode("utf-8") # Required by downstream text packet builder; no transcoding. # Prepare a self-contained matching registry with unchanged reference bytes. supports = {"source-inventory.json": source_bytes, "source-registry.json": registry_bytes} relocated = copy.deepcopy(registry) for task in relocated["tasks"]: for field, directory in (("public_packet", "public"), ("assessor_guide", "assessor-only")): asset = task[field] relative = directory+"/"+Path(asset["path"]).name content = read_source(registry_path.parent/asset["path"], asset["sha256"]) destination = "registry/"+relative require(destination not in supports or supports[destination] == content, "registry output collision") supports[destination] = content asset["path"] = relative common = relocated["common_conventions"] common_bytes = read_source(registry_path.parent/common["path"], common["sha256"]) require("registry/assessor-only/README.md" not in supports or supports["registry/assessor-only/README.md"] == common_bytes, "registry/conventions output collision") supports["registry/assessor-only/README.md"] = common_bytes common["path"] = "assessor-only/README.md" supports["registry/task-registry.json"] = json_bytes(relocated) seal_id = "seal-"+uuid.uuid4().hex archived_records = [] for rid, row in records.items(): archived_records.append(dict(record_id=rid, kind=row["kind"], source_path=str((inventory_path.parent/row["path"]).resolve()), path="records/record-"+uuid.uuid4().hex, sha256=sha(snapshots[rid]), bytes=len(snapshots[rid]))) index = {r["record_id"]: r for r in archived_records} inventory = dict(archive_inventory_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, batch_id=source["batch_id"], seal_record_id=seal_id, commit_file="packet-build-manifest.json", state="prepared; sealed only by the final commit file", records=archived_records, support_files=[dict(path=p, sha256=sha(c), bytes=len(c)) for p, c in sorted(supports.items())], runs=copy.deepcopy(source["runs"]), unassessable_runs=[]) for run in runs.values(): if run["endpoint_selection"]["artifact_id"] is None: inventory["unassessable_runs"].append(dict(run_id=run["run_id"], condition_id=run["condition_id"], task_id=run["task_id"], repetition_id=run["repetition_id"], reason=run["endpoint_selection"]["basis"])) inventory_content = json_bytes(inventory) inventory_hash = sha(inventory_content) # Reserve a new destination exclusively. Failures retain an unsealed partial # archive for inspection; existing outputs and source files are never repaired. output.mkdir() for row in archived_records: write_bytes(output/row["path"], snapshots[row["record_id"]]) for relative, content in supports.items(): write_bytes(output/relative, content) write_bytes(output/"archive-inventory.json", inventory_content) write_bytes(output/"archive-inventory.sha256", (inventory_hash+"\n").encode()) verify_payloads(output, inventory, inventory_hash) registry_data(relocated, output/"registry") sealed_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") packet = dict(build_manifest_version="1.0", batch_id=source["batch_id"], seal_record_id=seal_id, batch_sealed=True, sealed_at_utc=sealed_at, independent_rater_ids=source["independent_rater_ids"], conditions=source["conditions"], runs=[]) for run in runs.values(): endpoint = run["endpoint_selection"]["artifact_id"] if endpoint is None: continue # Explicitly retained in inventory.unassessable_runs; no imputation. item = index[endpoint] packet["runs"].append({k: run[k] for k in ("run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id")}) packet["runs"][-1].update(sealed=True, sealed_at_utc=sealed_at, final_artifact=dict(artifact_id=endpoint, path=item["path"], sha256=item["sha256"])) require(packet["runs"], "cannot publish an empty downstream manifest") packet_content = json_bytes(packet) require(parse_json(packet_content, "prepared packet manifest") == packet, "packet manifest serialization mismatch") # Recheck every archived payload immediately before publishing the seal. verify_payloads(output, inventory, inventory_hash) publish_manifest(output, packet_content) return dict(sealer_version="1.0", notice=NOTICE, archive=str(output), batch_sealed=True, packet_build_manifest=str(output/"packet-build-manifest.json"), relocated_registry=str(output/"registry"/"task-registry.json"), declared_runs=len(runs), assessable_runs=len(packet["runs"]), unassessable_runs=inventory["unassessable_runs"], declared_records=len(records)) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--inventory", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--registry", type=Path, default=KIT/"task-registry.json") args = parser.parse_args(argv) try: result = seal_inventory(args.inventory, args.output, args.registry) except KeyboardInterrupt: print(json.dumps(dict(error="interrupted; only the final packet-build-manifest.json commits a seal", notice=NOTICE)), file=sys.stderr) return 130 except (ValidationError, account_resources.LedgerError, OSError, KeyError, TypeError, ValueError, ArithmeticError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2)) return 0 if __name__ == "__main__": sys.exit(main())