#!/usr/bin/env python3 """Offline coordinator-only human audit of explicitly selected sealed typed memory. Development tooling; no automatic semantic judgments or planner feedback. """ import argparse from datetime import datetime, timezone import json import os from pathlib import Path import sys BASE = Path(__file__).resolve().parent sys.path.insert(0, str(BASE.parent/"run-sealing")) sys.path.insert(0, str(BASE.parent/"permitted-observations")) import summarize_run_outcomes as archives import assemble_observation as typed sealing = archives.sealing NOTICE = "Development memory-audit tooling, not empirical evidence; supplied human judgments only; coordinator-only, never planner input." JUDGMENTS = ("not_normative", "supported_normative", "unsupported_normative") IDENTITY = ("seal_record_id", "run_id", "snapshot_record_id", "snapshot_sha256", "entry_id", "entry_sha256") RECORD_FIELDS = (*IDENTITY, "judgment", "rationale", "evidence_locations", "reviewed_at_utc", "independent_judgments", "resolution_note") def read_json(path, label): content = Path(path).read_bytes() return sealing.parse_json(content, label), sealing.sha(content) def typed_snapshot(content): memory = typed.parse_json(content, "sealed typed memory") sealing.require(isinstance(memory, list), "snapshot must be a typed-memory replacement array") # An archive need not contain the original observation's ID catalog. Collect # syntactically valid references only to reuse the unchanged entry validator; # this does NOT establish original catalog membership or semantic support. catalogs = {k: set() for k in ("artifact_id", "source_id", "requirement_id")} for entry in memory: sealing.require(isinstance(entry, dict), "invalid typed memory entry") for reference in sealing.array(entry.get("evidence"), "entry evidence"): sealing.require(isinstance(reference, dict), "invalid evidence reference") for key in catalogs.keys() & reference.keys(): catalogs[key].add(typed.text(reference[key], "evidence identifier")) typed.validate_typed_memory(memory, catalogs, set(), len(memory)) return memory def load_audit(manifest_path, registry_path=None): manifest_path = Path(manifest_path).resolve() manifest, manifest_hash = read_json(manifest_path, "audit manifest") sealing.keys(manifest, ("audit_manifest_version", "audit_id", "independent_rater_ids", "adjudicator_id", "archives", "runs"), "audit manifest") sealing.require(manifest["audit_manifest_version"] == "1.0", "incompatible audit manifest version") sealing.text(manifest["audit_id"], "audit_id") sealing.strings(manifest["independent_rater_ids"], "independent rater IDs") raters = manifest["independent_rater_ids"] sealing.require(len(raters) == len(set(raters)) == 2, "declare two distinct independent raters") adjudicator = sealing.text(manifest["adjudicator_id"], "adjudicator_id") sealing.require(adjudicator not in raters, "adjudicator must be distinct from independent raters") paths = {} for descriptor in sealing.array(manifest["archives"], "archives"): sealing.keys(descriptor, ("archive_id", "path"), "archive descriptor") aid = sealing.text(descriptor["archive_id"], "archive_id") sealing.require(aid not in paths, "duplicate archive alias") paths[aid] = (manifest_path.parent/sealing.text(descriptor["path"], "archive path")).resolve() # Reuses complete sealing/hash/identity validation, including duplicate cells. collection = archives.summarize_archives(list(paths.values()), registry_path) provenance = {p["archive"]: p for p in collection["archives"]} declared_runs = {r["run_id"]: r for r in collection["runs"]} inventories = {aid: read_json(archives.inside(root, "archive-inventory.json"), "sealed inventory")[0] for aid, root in paths.items()} runs, entries = {}, {} for selection in sealing.array(manifest["runs"], "run selections"): sealing.keys(selection, ("archive_id", "seal_record_id", "run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id", "final_memory", "unavailability_reason"), "run selection") rid = sealing.text(selection["run_id"], "run_id") sealing.require(rid not in runs and rid in declared_runs, "duplicate/unknown selected run") aid = selection["archive_id"] sealing.require(aid in paths, "unknown archive alias") source = declared_runs[rid] seal = provenance[str(paths[aid])] sealing.require(selection["seal_record_id"] == seal["seal_record_id"] == source["archive_seal_record_id"], "selected archive/run seal identity mismatch") sealing.require(all(selection[k] == source[k] for k in ("condition_id", "task_id", "task_version", "rubric_version", "repetition_id")), "selected run identity/task/rubric mismatch") run = dict(selection, sealed_at_utc=seal["sealed_at_utc"], entry_keys=[], entry_count=None) snapshot = selection["final_memory"] if snapshot is None: sealing.text(selection["unavailability_reason"], "unavailable final-memory explanation") else: sealing.keys(snapshot, ("record_id", "sha256", "selection_basis"), "final memory selection") sealing.text(snapshot["selection_basis"], "explicit final-memory selection basis") sealing.require(selection["unavailability_reason"] is None, "available memory cannot have an unavailability reason") sealing.digest(snapshot["sha256"], "snapshot sha256") inventory = inventories[aid] original_run = next(r for r in inventory["runs"] if r["run_id"] == rid) records = {r["record_id"]: r for r in inventory["records"]} sid = sealing.text(snapshot["record_id"], "snapshot record ID") sealing.require(sid in original_run["record_ids"] and records[sid]["kind"] == "memory_snapshot", "snapshot must designate this run's sealed memory record") record = records[sid] content = archives.inside(paths[aid], record["path"]).read_bytes() sealing.require(snapshot["sha256"] == record["sha256"] == sealing.sha(content), "selected snapshot hash mismatch") memory = typed_snapshot(content) run["entry_count"] = len(memory) for entry in memory: key = (rid, entry["id"]) identity = dict(seal_record_id=selection["seal_record_id"], run_id=rid, snapshot_record_id=sid, snapshot_sha256=snapshot["sha256"], entry_id=entry["id"], entry_sha256=typed.sha(typed.canonical(entry).encode("utf-8"))) entries[key] = identity run["entry_keys"].append(key) runs[rid] = run sealing.require(set(runs) == set(declared_runs), "declare a final snapshot or explicit unavailability for every supplied archived run") return dict(manifest=manifest, manifest_sha256=manifest_hash, runs=runs, entries=entries, archives=collection["archives"]) def blank_form(audit, kind, assessor): records = [dict(identity, judgment=None, rationale=None, evidence_locations=[], reviewed_at_utc=None, independent_judgments=None, resolution_note=None) for _, identity in sorted(audit["entries"].items())] return dict(assessment_form_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, audit_id=audit["manifest"]["audit_id"], audit_manifest_sha256=audit["manifest_sha256"], series_kind=kind, assessor_id=assessor, records=records) def write_forms(manifest_path, output, registry_path=None): audit = load_audit(manifest_path, registry_path) output = Path(output).absolute() sealing.require(not os.path.lexists(output), "refusing existing forms destination") sealing.require(output.parent.is_dir(), "forms parent must already exist") output = output.resolve() sealing.require(all(output != Path(p["archive"]) and Path(p["archive"]) not in output.parents for p in audit["archives"]), "forms destination must be outside sealed archives") output.mkdir() for i, rater in enumerate(audit["manifest"]["independent_rater_ids"], 1): (output/f"independent-{i}.json").write_bytes(sealing.json_bytes(blank_form(audit, "independent", rater))) (output/"adjudication.json").write_bytes(sealing.json_bytes(blank_form(audit, "adjudicated", audit["manifest"]["adjudicator_id"]))) return dict(notice=NOTICE, defaults_unfrozen=True, coordinator_only=True, blank_form_count=3, audit_manifest_sha256=audit["manifest_sha256"]) def load_assessments(audit, paths): raters = audit["manifest"]["independent_rater_ids"] expected = [("independent", r) for r in raters]+[("adjudicated", audit["manifest"]["adjudicator_id"])] series = {key: {} for key in expected} seen, sources = set(), [] now = datetime.now(timezone.utc) for path in paths: form, source_hash = read_json(path, "human assessment form") sealing.keys(form, ("assessment_form_version", "notice", "coordinator_only", "defaults_unfrozen", "audit_id", "audit_manifest_sha256", "series_kind", "assessor_id", "records"), "assessment form") sealing.require(form["assessment_form_version"] == "1.0", "incompatible assessment form version") sealing.require(form["notice"] == NOTICE and form["coordinator_only"] is True and form["defaults_unfrozen"] is True, "assessment form must retain coordinator-only development labels") sealing.require(form["audit_id"] == audit["manifest"]["audit_id"] and form["audit_manifest_sha256"] == audit["manifest_sha256"], "assessment audit identity/hash mismatch") series_key = (form["series_kind"], form["assessor_id"]) sealing.require(series_key in series and series_key not in seen, "unknown/duplicate assessment series") seen.add(series_key) sources.append(dict(path=str(Path(path).resolve()), sha256=source_hash, series_kind=series_key[0], assessor_id=series_key[1])) for row in sealing.array(form["records"], "assessment records"): sealing.keys(row, RECORD_FIELDS, "entry assessment") key = (sealing.text(row["run_id"], "run_id"), sealing.text(row["entry_id"], "entry_id")) sealing.require(key in audit["entries"] and key not in series[series_key], "unknown/duplicate assessed entry") sealing.require(all(row[k] == audit["entries"][key][k] for k in IDENTITY), "snapshot/entry identity or hash mismatch") judgment = row["judgment"] sealing.require(judgment is None or judgment in JUDGMENTS, "invalid human judgment") locations = sealing.array(row["evidence_locations"], "human evidence locations") for location in locations: sealing.text(location, "human evidence location") if judgment is None: sealing.require(row["reviewed_at_utc"] is None and row["independent_judgments"] is None and row["resolution_note"] is None, "missing judgment cannot be marked completed/resolved") sealing.require(row["rationale"] is None or isinstance(row["rationale"], str), "invalid pending rationale") else: sealing.text(row["rationale"], "human judgment rationale") sealing.require(locations, "judgment requires human evidence locations") stamp = sealing.utc(row["reviewed_at_utc"], "reviewed_at_utc") sealing.require(sealing.utc(audit["runs"][key[0]]["sealed_at_utc"], "seal time") <= stamp <= now, "human assessment must occur after sealing and not in the future") if series_key[0] == "independent": sealing.require(row["independent_judgments"] is None and row["resolution_note"] is None, "independent records cannot contain adjudication fields") series[series_key][key] = row # Validate adjudication after all independent files, independent of file order. for key, row in series[expected[-1]].items(): if row["judgment"] is None: continue sealing.keys(row["independent_judgments"], raters, "adjudication's independent judgments") sealing.text(row["resolution_note"], "adjudication resolution note") for rater in raters: independent = series[("independent", rater)].get(key) sealing.require(independent is not None and independent["judgment"] is not None, "adjudication requires both independent judgments") sealing.require(row["independent_judgments"][rater] == independent["judgment"], "stale/mismatched adjudication basis") sealing.require(sealing.utc(row["reviewed_at_utc"], "adjudication time") >= sealing.utc(independent["reviewed_at_utc"], "independent time"), "adjudication predates independent assessment") return series, sources def summarize_group(runs, judgments): available = [r for r in runs if r["entry_count"] is not None] available_entries = sum(r["entry_count"] for r in available) run_rows = [] for run in runs: keys = run["entry_keys"] assessed = [judgments[k] for k in keys if k in judgments and judgments[k]["judgment"] is not None] unsupported = sum(row["judgment"] == "unsupported_normative" for row in assessed) missing = None if run["entry_count"] is None else len(keys)-len(assessed) complete = missing == 0 run_rows.append(dict(run_id=run["run_id"], entry_count=run["entry_count"], assessed_entries=len(assessed), missing_judgments=missing, unsupported_normative_entries=unsupported, affected=True if unsupported else (False if complete else None), memory_audit_complete=complete)) assessed = sum(r["assessed_entries"] for r in run_rows) unsupported = sum(r["unsupported_normative_entries"] for r in run_rows) missing = available_entries-assessed unavailable = len(runs)-len(available) complete = unavailable == 0 and missing == 0 affected = sum(r["affected"] is True for r in run_rows) return dict(declared_runs=len(runs), available_snapshot_runs=len(available), unavailable_memory_runs=unavailable, empty_memory_runs=sum(r["entry_count"] == 0 for r in available), available_snapshot_entry_count=available_entries, assessed_memory_entries=assessed, missing_judgment_entries=missing, judgment_coverage=dict(numerator=assessed, denominator=available_entries, rate=None if available_entries == 0 else assessed/available_entries), unsupported_normative_entries=unsupported, unsupported_normative_rate=dict(numerator=unsupported, denominator=assessed, rate=None if assessed == 0 else unsupported/assessed), confirmed_affected_runs=affected, fully_assessed_available_runs=sum(r["memory_audit_complete"] for r in run_rows), available_runs_with_missing_judgments=sum(r["missing_judgments"] is not None and r["missing_judgments"] > 0 for r in run_rows), complete_declared_run_coverage=complete, affected_run_rate_over_all_declared=dict(numerator=affected, denominator=len(runs), rate=affected/len(runs) if complete and runs else None, withheld_for_missing_coverage=not complete), runs=run_rows) def summarize(manifest_path, assessment_paths, registry_path=None): audit = load_audit(manifest_path, registry_path) series, sources = load_assessments(audit, assessment_paths) conditions = sorted({run["condition_id"] for run in audit["runs"].values()}) def summary(rows): return dict(overall=summarize_group(list(audit["runs"].values()), rows), by_condition={c: summarize_group([r for r in audit["runs"].values() if r["condition_id"] == c], rows) for c in conditions}) return dict(memory_drift_report_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, audit_id=audit["manifest"]["audit_id"], audit_manifest_sha256=audit["manifest_sha256"], independent_rater_ids=audit["manifest"]["independent_rater_ids"], adjudicator_id=audit["manifest"]["adjudicator_id"], selection_policy="one explicitly designated final typed-memory snapshot per run; no historical inference", judgment_policy="manual joint classification; no inference from kind, status, or references", archives=audit["archives"], assessment_sources=sources, independent={r: summary(series[("independent", r)]) for r in audit["manifest"]["independent_rater_ids"]}, adjudicated=summary(series[("adjudicated", audit["manifest"]["adjudicator_id"])])) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="command", required=True) for command in ("forms", "summarize"): child = sub.add_parser(command) child.add_argument("--manifest", type=Path, required=True) child.add_argument("--registry", type=Path) if command == "forms": child.add_argument("--output", type=Path, required=True) else: child.add_argument("--assessments", type=Path, nargs="*", default=[]) args = parser.parse_args(argv) try: result = (write_forms(args.manifest, args.output, args.registry) if args.command == "forms" else summarize(args.manifest, args.assessments, args.registry)) except (sealing.ValidationError, typed.AssemblyError, sealing.account_resources.LedgerError, OSError, KeyError, TypeError, ValueError, ArithmeticError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())