#!/usr/bin/env python3 """Offline human designation and assessment of critical public-requirement violations. Untested development tooling; no criticality inference or automatic judgments. """ import argparse from datetime import datetime, timezone import json from pathlib import Path import random import re import sys import uuid BASE = Path(__file__).resolve().parent sys.path.insert(0, str(BASE.parent/"run-sealing")) sys.path.insert(0, str(BASE.parent/"permitted-observations")) import summarize_run_outcomes as archived import assemble_observation as observation sealing = archived.sealing import build_assessment_packets as packets import score_forms as scoring NOTICE = "Development critical-violation tooling, not empirical evidence; defaults unfrozen; human designation and judgments only." ARTIFACT_ID = ("artifact_id", "artifact_sha256", "task_id", "task_version", "rubric_version", "family_id", "source_packet_sha256", "designation_sha256") REQUIREMENT_ID = ("requirement_id", "requirement_location", "source_locations") RATING_FIELDS = (*ARTIFACT_ID, *REQUIREMENT_ID, "violation", "invented_restriction", "reason", "artifact_evidence", "artifact_scope", "completed_at_utc", "independent_judgments", "resolution_note") INSTRUCTIONS = """# Human critical public-requirement assessment Untested development tooling; defaults unfrozen. Assess only the explicitly designated public requirements against the supplied final artifact and immutable public packet. Criticality was designated by a human, not inferred from failures. Exact public requirements remain the authority: add no hidden rules, stricter thresholds, or new critical requirements. Criticality is not a new requirement. Set violation to integer 1 for a violated designated requirement, or 0 for no violation. Independently record invented_restriction as 1 only when the violation includes an unauthorized restriction against that same public requirement; use 0 otherwise. A flag of 1 requires violation=1. Both start null; unresolved judgments stay null. An invented restriction is not an extra scoring item. Count each artifact/requirement once, even with several failures or restrictions. Record a reason and artifact_scope describing the material reviewed. Quote exact artifact spans as artifact_evidence where possible. Spans use zero-based Unicode code-point [start,end) offsets in unchanged UTF-8 text, including CR/LF/BOM. An omission can have no quoted span, but must explain its absence and reviewed scope. Valid source pointers alone do not prove either criticality or a violation. Rate independently, then lock the form before sharing. A third human adjudicates both locked judgments in a separate form; agreement is never auto-copied. Do not send any assessments back to adaptation. Do not open coordinator mappings, prompts, memory, usage records or primary criterion scores. Opaque IDs cannot remove intrinsic method disclosures in artifact text. Report issues rather than edit it. Directory separation does not enforce confidentiality or prove blinding. """ def read_json(path, label): content = Path(path).read_bytes() return sealing.parse_json(content, label), content def span(reference, text, label): sealing.keys(reference, ("start", "end", "text"), label) start, end = reference["start"], reference["end"] sealing.require(type(start) is int and type(end) is int and 0 <= start < end <= len(text), label+": invalid span bounds") sealing.text(reference["text"], label+" quote") sealing.require(text[start:end] == reference["text"], label+": quote does not match exact text") def public_catalog(packet, task_id): public_ids, source_ids = observation.task_identifiers(packet, task_id) requirements = {} for match in re.finditer(r"^- \*\*(R[1-9][0-9]*):\*\*", packet, re.MULTILINE): boundary = re.search(r"^(?:- \*\*R[1-9][0-9]*:\*\*|## )", packet[match.end():], re.MULTILINE) end = len(packet) if boundary is None else match.end()+boundary.start() end = match.start()+len(packet[match.start():end].rstrip("\r\n")) requirements[match.group(1)] = dict(start=match.start(), end=end, text=packet[match.start():end]) headings = list(re.finditer(r"^### (S[1-9][0-9]*)(?:\s|$)[^\r\n]*", packet, re.MULTILINE)) sources = {} for heading in headings: boundary = re.search(r"^#{1,3} ", packet[heading.end():], re.MULTILINE) sources[heading.group(1)] = (heading.start(), len(packet) if boundary is None else heading.end()+boundary.start()) sealing.require(set(requirements) == public_ids and set(sources) == source_ids, "incompatible public packet identifier structure") return requirements, sources def validate_designation(content, task, packet_bytes, allowed_ids): data = sealing.parse_json(content, "human critical designation") sealing.keys(data, ("designation_version", "task_id", "task_version", "rubric_version", "source_packet_sha256", "review_status", "designator_id", "completed_at_utc", "note", "requirements"), "critical designation") sealing.require(data["designation_version"] == "1.0", "incompatible designation version") sealing.require(all(data[k] == task[k] for k in ("task_id", "task_version", "rubric_version")) and data["source_packet_sha256"] == sealing.sha(packet_bytes), "designation task/source identity/hash/version mismatch") sealing.require(data["review_status"] in ("reviewed", "unfinished"), "invalid designation review status") sealing.text(data["designator_id"], "human designator") sealing.text(data["note"], "designation scope/limitations note") if data["review_status"] == "reviewed": sealing.require(sealing.utc(data["completed_at_utc"], "designation completion") <= datetime.now(timezone.utc), "future designation review") else: sealing.require(data["completed_at_utc"] is None, "unfinished designation cannot claim completed review") packet = packet_bytes.decode("utf-8") requirements, sources = public_catalog(packet, task["task_id"]) seen = set() for row in sealing.array(data["requirements"], "designated public requirements"): sealing.keys(row, REQUIREMENT_ID, "designated public requirement") rid = sealing.text(row["requirement_id"], "public requirement ID") sealing.require(rid in requirements and rid in allowed_ids, "unknown public requirement ID") sealing.require(rid not in seen, "duplicate designated requirement identity") seen.add(rid) span(row["requirement_location"], packet, "public requirement location") sealing.require(row["requirement_location"] == requirements[rid], "requirement location must quote the whole matching public requirement") refs = set() for ref in sealing.array(row["source_locations"], "public source locations"): sealing.keys(ref, ("source_id", "start", "end", "text"), "public source location") sid = ref["source_id"] sealing.require(sid in sources, "unknown authorized source ID") span({k: ref[k] for k in ("start", "end", "text")}, packet, "public source location") lo, hi = sources[sid] sealing.require(lo <= ref["start"] < ref["end"] <= hi, "source reference lies outside named source section") identity = (sid, ref["start"], ref["end"]) sealing.require(identity not in refs, "duplicate public source reference") refs.add(identity) return data def load_selection(manifest_path, registry_path=None, source_parent=None, designation_copies=None): manifest_path = Path(manifest_path).resolve() manifest, manifest_bytes = read_json(manifest_path, "critical assessment selection") parent = source_parent or manifest_path.parent sealing.keys(manifest, ("critical_manifest_version", "assessment_id", "independent_rater_ids", "adjudicator_id", "archives", "artifacts", "designations"), "critical selection") sealing.require(manifest["critical_manifest_version"] == "1.0", "incompatible critical selection version") sealing.text(manifest["assessment_id"], "assessment ID") raters = manifest["independent_rater_ids"] sealing.strings(raters, "independent raters") sealing.require(len(raters) == len(set(raters)) == 2, "declare two distinct independent raters") sealing.require(sealing.text(manifest["adjudicator_id"], "adjudicator") not in raters, "adjudicator must be distinct") roots = {} for row in sealing.array(manifest["archives"], "archives"): sealing.keys(row, ("archive_id", "path"), "archive descriptor") aid = sealing.text(row["archive_id"], "archive alias") sealing.require(aid not in roots, "duplicate archive alias") roots[aid] = (parent/sealing.text(row["path"], "archive path")).resolve() collection = archived.summarize_archives(list(roots.values()), registry_path) registry_path = Path(registry_path or sealing.KIT/"task-registry.json").resolve() registry, _ = read_json(registry_path, "registry") tasks, _, mapping = sealing.registry_data(registry, registry_path.parent) endpoints, commits = {}, {} for aid, root in roots.items(): commit, _ = read_json(archived.inside(root, "packet-build-manifest.json"), "archive commit") commits[aid] = commit endpoints.update({(aid, r["run_id"]): r for r in commit["runs"]}) selected, seen = [], set() for row in sealing.array(manifest["artifacts"], "selected artifacts"): sealing.keys(row, ("archive_id", "seal_record_id", "run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id", "artifact_id", "artifact_sha256"), "selected artifact") rid, aid = sealing.text(row["run_id"], "run ID"), row["archive_id"] sealing.require(rid not in seen and (aid, rid) in endpoints, "duplicate selection or run lacks a sealed designated final artifact") seen.add(rid) run = endpoints[(aid, rid)] sealing.require(row["seal_record_id"] == commits[aid]["seal_record_id"] and all(row[k] == run[k] for k in ("condition_id", "task_id", "task_version", "rubric_version", "repetition_id")), "selected run identity/version mismatch") final = run["final_artifact"] sealing.require(row["artifact_id"] == final["artifact_id"] and row["artifact_sha256"] == final["sha256"], "final artifact identity/hash mismatch") content = packets.hashed_bytes(archived.inside(roots[aid], final["path"]), final["sha256"]) content.decode("utf-8") selected.append(dict(source=row, content=content, sealed_at_utc=commits[aid]["sealed_at_utc"], family_id=tasks[row["task_id"]]["family_id"])) sealing.require(selected, "select at least one final artifact") needed = {r["source"]["task_id"] for r in selected} designations, copies, references = {}, {}, {} for row in sealing.array(manifest["designations"], "task designations"): sealing.keys(row, ("task_id", "designation", "unavailability_reason"), "task designation descriptor") tid = row["task_id"] sealing.require(tid in needed and tid not in designations, "unknown/duplicate task designation") public = tasks[tid]["public_packet"] packet = packets.hashed_bytes(registry_path.parent/public["path"], public["sha256"]) public_catalog(packet.decode("utf-8"), tid) references[tid] = packet value = None if row["designation"] is None: sealing.text(row["unavailability_reason"], "unavailable designation explanation") else: descriptor = row["designation"] sealing.keys(descriptor, ("path", "sha256"), "designation file") sealing.text(descriptor["path"], "designation path") sealing.digest(descriptor["sha256"], "designation source hash") sealing.require(row["unavailability_reason"] is None, "available designation has unavailability explanation") content = designation_copies[tid] if designation_copies is not None else packets.hashed_bytes(parent/descriptor["path"], descriptor["sha256"]) sealing.require(sealing.sha(content) == descriptor["sha256"], "designation source hash mismatch") value = validate_designation(content, tasks[tid], packet, {rid for _, rid in mapping}) copies[tid] = content designations[tid] = value sealing.require(set(designations) == needed, "declare reviewed, unfinished, or unavailable designation for every selected task") if designation_copies is not None: sealing.require(set(designation_copies) == set(copies), "copied designation coverage mismatch") return dict(manifest=manifest, manifest_bytes=manifest_bytes, manifest_hash=sealing.sha(manifest_bytes), registry=registry, selected=sorted(selected, key=lambda r: r["source"]["run_id"]), designations=designations, designation_copies=copies, references=references, archives=collection["archives"], unselected_archived_run_ids=sorted(r["run_id"] for r in collection["runs"] if r["run_id"] not in seen)) def public_designation(tid, design): value = design["designations"][tid] task = next(t for t in design["registry"]["tasks"] if t["task_id"] == tid) return dict(designation_version="1.0", notice=NOTICE, defaults_unfrozen=True, task_id=tid, task_version=task["task_version"], rubric_version=task["rubric_version"], source_packet_sha256=sealing.sha(design["references"][tid]), designation_state="unavailable" if value is None else value["review_status"], requirements=[] if value is None else value["requirements"]) def build(manifest_path, rater_dir, coordinator_dir, registry_path=None, seed=0): sealing.require(type(seed) is int, "presentation seed must be integer") rater_dir, coordinator_dir = packets.output_path(rater_dir), packets.output_path(coordinator_dir) sealing.require(packets.disjoint(rater_dir, coordinator_dir), "coordinator and rater outputs must be disjoint") design = load_selection(manifest_path, registry_path) for output in (rater_dir, coordinator_dir): sealing.require(all(output != Path(p["archive"]) and Path(p["archive"]) not in output.parents for p in design["archives"]), "outputs must stay outside sealed archives") issued = set() def opaque(prefix): value = prefix+"-"+uuid.uuid4().hex sealing.require(value not in issued, "opaque ID collision") issued.add(value) return value raters = {r: opaque("rater") for r in design["manifest"]["independent_rater_ids"]} adjudicator = opaque("rater") key = dict(critical_key_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, batch_id=opaque("batch"), seed=seed, manifest_parent=str(Path(manifest_path).resolve().parent), source_manifest_sha256=design["manifest_hash"], registry_signature_sha256=sealing.sha(archived.registry_signature(design["registry"])), independent_rater_aliases=raters, adjudicator_alias=adjudicator, artifacts=[], designation_copies=[], forms=[]) public = {tid: public_designation(tid, design) for tid in design["designations"]} public_bytes = {tid: sealing.json_bytes(value) for tid, value in public.items()} artifact_bytes = {} for item in design["selected"]: source, aid = item["source"], opaque("artifact") tid = source["task_id"] state = public[tid]["designation_state"] bound = item["sealed_at_utc"] if state == "reviewed": bound = max((bound, design["designations"][tid]["completed_at_utc"]), key=lambda x: sealing.utc(x, "assessment lower bound")) row = dict(artifact_id=aid, artifact_sha256=source["artifact_sha256"], task_id=tid, task_version=source["task_version"], rubric_version=source["rubric_version"], family_id=item["family_id"], source_packet_sha256=sealing.sha(design["references"][tid]), designation_sha256=sealing.sha(public_bytes[tid]), source_run_id=source["run_id"], condition_id=source["condition_id"], repetition_id=source["repetition_id"], sealed_at_utc=item["sealed_at_utc"], rating_not_before_utc=bound, designation_state=state, requirements=public[tid]["requirements"]) key["artifacts"].append(row) artifact_bytes[aid] = item["content"] rng, bundles = random.Random(seed), [] for kind, assessor in [("independent", r) for r in raters.values()]+[("adjudicated", adjudicator)]: ordered = list(key["artifacts"]) rng.shuffle(ordered) form = dict(critical_form_version="1.0", notice=NOTICE, defaults_unfrozen=True, batch_id=key["batch_id"], form_id=opaque("form"), assessor_id=assessor, series_kind=kind, locked_at_utc=None, ratings=[]) for artifact in ordered: for requirement in artifact["requirements"]: form["ratings"].append(dict({k: artifact[k] for k in ARTIFACT_ID}, **requirement, violation=None, invented_restriction=None, reason=None, artifact_evidence=[], artifact_scope=None, completed_at_utc=None, independent_judgments=None, resolution_note=None)) key["forms"].append(dict(assessor_id=assessor, form_id=form["form_id"], series_kind=kind, artifact_order=[a["artifact_id"] for a in ordered])) index = dict(packet_version="1.0", notice=NOTICE, defaults_unfrozen=True, batch_id=key["batch_id"], assessor_id=assessor, artifacts=[dict({k: a[k] for k in (*ARTIFACT_ID, "designation_state")}, artifact_path="artifacts/"+a["artifact_id"]+".txt", source_packet_path="public/"+a["task_id"]+".txt", designation_path="critical-requirements/"+a["task_id"]+".json") for a in ordered]) bundles.append((assessor, form, index)) for i, (tid, content) in enumerate(sorted(design["designation_copies"].items())): key["designation_copies"].append(dict(task_id=tid, path=f"designations/{i}.json", sha256=sealing.sha(content))) rater_dir.mkdir() coordinator_dir.mkdir() for assessor, form, index in bundles: root = rater_dir/assessor root.mkdir() packets.write_json(root/"rating-form.json", form) packets.write_json(root/"packet-index.json", index) packets.write_bytes(root/"INSTRUCTIONS.md", INSTRUCTIONS.encode("utf-8")) for aid, content in artifact_bytes.items(): packets.write_bytes(root/"artifacts"/(aid+".txt"), content) for tid, content in design["references"].items(): packets.write_bytes(root/"public"/(tid+".txt"), content) packets.write_bytes(root/"critical-requirements"/(tid+".json"), public_bytes[tid]) packets.write_bytes(coordinator_dir/"source-manifest.json", design["manifest_bytes"]) for row in key["designation_copies"]: packets.write_bytes(coordinator_dir/row["path"], design["designation_copies"][row["task_id"]]) packets.write_json(coordinator_dir/"critical-key.json", key) return dict(notice=NOTICE, defaults_unfrozen=True, artifact_count=len(artifact_bytes), blank_form_count=3) def same(left, right): """Exact JSON value identity, including integer versus boolean span types.""" return json.dumps(left, sort_keys=True, allow_nan=False) == json.dumps(right, sort_keys=True, allow_nan=False) def load_key(key_path, registry_path=None): key_path = Path(key_path).resolve() key, content = read_json(key_path, "critical coordinator key") sealing.keys(key, ("critical_key_version", "notice", "coordinator_only", "defaults_unfrozen", "batch_id", "seed", "manifest_parent", "source_manifest_sha256", "registry_signature_sha256", "independent_rater_aliases", "adjudicator_alias", "artifacts", "designation_copies", "forms"), "critical coordinator key") sealing.require(key["critical_key_version"] == "1.0", "incompatible critical key version") sealing.require(key["notice"] == NOTICE and key["coordinator_only"] is True and key["defaults_unfrozen"] is True, "retain coordinator development labels") sealing.require(type(key["seed"]) is int, "invalid presentation seed") parent = Path(sealing.text(key["manifest_parent"], "original manifest parent")) sealing.require(parent.is_absolute(), "manifest parent must be absolute") manifest_path = archived.inside(key_path.parent, "source-manifest.json") packets.hashed_bytes(manifest_path, key["source_manifest_sha256"]) copies, paths = {}, set() for descriptor in sealing.array(key["designation_copies"], "cached designations"): sealing.keys(descriptor, ("task_id", "path", "sha256"), "cached designation") tid = sealing.text(descriptor["task_id"], "cached task ID") sealing.require(tid not in copies and descriptor["path"] not in paths, "duplicate cached designation identity/path") paths.add(descriptor["path"]) copies[tid] = packets.hashed_bytes(archived.inside(key_path.parent, descriptor["path"]), descriptor["sha256"]) design = load_selection(manifest_path, registry_path, source_parent=parent, designation_copies=copies) sealing.require(design["manifest_hash"] == key["source_manifest_sha256"], "source selection changed during validation") sealing.require(key["registry_signature_sha256"] == sealing.sha(archived.registry_signature(design["registry"])), "incompatible registry signature") aliases = key["independent_rater_aliases"] sealing.keys(aliases, design["manifest"]["independent_rater_ids"], "independent rater mapping") issued = set() def opaque(value, prefix): sealing.require(isinstance(value, str) and re.fullmatch(prefix+r"-[0-9a-f]{32}", value) is not None and value not in issued, "invalid/duplicate opaque identity") issued.add(value) opaque(key["batch_id"], "batch") for alias in [*aliases.values(), key["adjudicator_alias"]]: opaque(alias, "rater") artifacts, expectations = {}, {} rows = sealing.array(key["artifacts"], "key artifacts") sealing.require(len(rows) == len(design["selected"]), "key artifact coverage mismatch") for row, item in zip(rows, design["selected"]): sealing.keys(row, (*ARTIFACT_ID, "source_run_id", "condition_id", "repetition_id", "sealed_at_utc", "rating_not_before_utc", "designation_state", "requirements"), "key artifact") source = item["source"] tid = source["task_id"] public = public_designation(tid, design) state = public["designation_state"] bound = item["sealed_at_utc"] if state == "reviewed": bound = max((bound, design["designations"][tid]["completed_at_utc"]), key=lambda x: sealing.utc(x, "assessment lower bound")) sealing.require(row["source_run_id"] == source["run_id"] and all(row[k] == source[k] for k in ("task_id", "task_version", "rubric_version", "artifact_sha256", "condition_id", "repetition_id")), "key source identity/hash mismatch") sealing.require(row["family_id"] == item["family_id"] and row["source_packet_sha256"] == public["source_packet_sha256"] and row["designation_sha256"] == sealing.sha(sealing.json_bytes(public)) and row["designation_state"] == state and same(row["requirements"], public["requirements"]) and row["sealed_at_utc"] == item["sealed_at_utc"] and row["rating_not_before_utc"] == bound, "key public designation/source/time mismatch") opaque(row["artifact_id"], "artifact") artifacts[row["artifact_id"]] = dict(row, artifact_text=item["content"].decode("utf-8")) for requirement in row["requirements"]: identity = dict({k: row[k] for k in ARTIFACT_ID}, **requirement) expectations[(row["artifact_id"], requirement["requirement_id"])] = identity expected_assessors = {r: "independent" for r in aliases.values()} | {key["adjudicator_alias"]: "adjudicated"} specs = {} for spec in sealing.array(key["forms"], "key forms"): sealing.keys(spec, ("assessor_id", "form_id", "series_kind", "artifact_order"), "key form") assessor = spec["assessor_id"] sealing.require(assessor in expected_assessors and assessor not in specs and spec["series_kind"] == expected_assessors[assessor], "unknown/duplicate key form") opaque(spec["form_id"], "form") sealing.strings(spec["artifact_order"], "artifact order") sealing.require(len(spec["artifact_order"]) == len(artifacts) and set(spec["artifact_order"]) == set(artifacts), "key form coverage mismatch") specs[assessor] = spec sealing.require(set(specs) == set(expected_assessors), "missing key form") return key, sealing.sha(content), design, artifacts, expectations, specs def load_ratings(paths, key, artifacts, expectations, specs): series, locks, seen, provenance = {r: {} for r in specs}, {}, set(), [] now = datetime.now(timezone.utc) for path in paths: form, content = read_json(path, "human critical rating form") sealing.keys(form, ("critical_form_version", "notice", "defaults_unfrozen", "batch_id", "form_id", "assessor_id", "series_kind", "locked_at_utc", "ratings"), "critical rating form") sealing.require(form["critical_form_version"] == "1.0", "incompatible critical form version") assessor = form["assessor_id"] sealing.require(assessor in specs and assessor not in seen, "unknown/duplicate assessment series") spec = specs[assessor] sealing.require(form["notice"] == NOTICE and form["defaults_unfrozen"] is True, "retain development form labels") sealing.require(form["batch_id"] == key["batch_id"] and all(form[k] == spec[k] for k in ("form_id", "series_kind")), "form batch/series identity mismatch") lock = None if form["locked_at_utc"] is None else sealing.utc(form["locked_at_utc"], "form lock") sealing.require(lock is None or lock <= now, "future form lock") locks[assessor] = lock seen.add(assessor) provenance.append(dict(path=str(Path(path).resolve()), sha256=sealing.sha(content), assessor_id=assessor, series_kind=form["series_kind"])) for row in sealing.array(form["ratings"], "critical ratings"): sealing.keys(row, RATING_FIELDS, "critical requirement rating") cell = (row["artifact_id"], row["requirement_id"]) sealing.require(cell in expectations and cell not in series[assessor], "unknown/duplicate assessed requirement identity") sealing.require(same({k: row[k] for k in (*ARTIFACT_ID, *REQUIREMENT_ID)}, expectations[cell]), "rating artifact/requirement/source/designation identity or hash mismatch") violation = scoring.binary(row["violation"], "critical violation") flag = scoring.binary(row["invented_restriction"], "invented restriction flag") for evidence in sealing.array(row["artifact_evidence"], "artifact evidence"): span(evidence, artifacts[row["artifact_id"]]["artifact_text"], "artifact evidence") if violation is None: sealing.require(flag is None and row["completed_at_utc"] is None and row["independent_judgments"] is None and row["resolution_note"] is None, "missing violation judgment marked completed/classified") for field in ("reason", "artifact_scope"): sealing.require(row[field] is None or isinstance(row[field], str), "invalid pending assessment note") else: sealing.require(flag is not None and flag <= violation, "invented restriction requires an assessed violation; complete both binary fields") sealing.text(row["reason"], "human violation rationale") sealing.text(row["artifact_scope"], "reviewed artifact scope/absence evidence") stamp = sealing.utc(row["completed_at_utc"], "assessment completion") sealing.require(lock is not None and sealing.utc(artifacts[row["artifact_id"]]["rating_not_before_utc"], "assessment lower bound") <= stamp <= lock, "judgment must follow sealing/designation review and precede form lock") if form["series_kind"] == "independent": sealing.require(row["independent_judgments"] is None and row["resolution_note"] is None, "independent form contains adjudication fields") series[assessor][cell] = row raters = list(key["independent_rater_aliases"].values()) for cell, row in series[key["adjudicator_alias"]].items(): if row["violation"] is None: continue sealing.keys(row["independent_judgments"], raters, "adjudication basis") sealing.text(row["resolution_note"], "adjudication resolution") for rater in raters: original = series[rater].get(cell) sealing.require(original is not None and original["violation"] is not None, "adjudication requires both independent judgments") basis = row["independent_judgments"][rater] sealing.keys(basis, ("violation", "invented_restriction"), "original judgment pair") sealing.require(same(basis, {k: original[k] for k in ("violation", "invented_restriction")}), "stale adjudication basis") sealing.require(sealing.utc(row["completed_at_utc"], "adjudication completion") >= locks[rater], "adjudication predates independent lock") return series, provenance def summarize_group(artifacts, ratings): rows = [] for artifact in artifacts: aid, state = artifact["artifact_id"], artifact["designation_state"] expected = [(aid, r["requirement_id"]) for r in artifact["requirements"]] assessed = [ratings[c] for c in expected if c in ratings and ratings[c]["violation"] is not None] count, flags = sum(r["violation"] for r in assessed), sum(r["invented_restriction"] for r in assessed) complete = state == "reviewed" and len(expected) == len(assessed) rows.append(dict(artifact_id=aid, designation_state=state, designated_requirements=None if state == "unavailable" else len(expected), assessed_critical_requirements=len(assessed), missing_judgments=None if state == "unavailable" else len(expected)-len(assessed), violated_critical_requirements=count, requirements_with_assessed_invented_restriction=flags, has_assessed_critical_violation=True if count else (False if complete else None), has_assessed_invented_restriction=True if flags else (False if complete else None), declared_coverage_complete=complete, critical_requirement_violation_rate=count/len(assessed) if complete and assessed else None)) known = sum(r["designated_requirements"] for r in rows if r["designated_requirements"] is not None) assessed = sum(r["assessed_critical_requirements"] for r in rows) violated = sum(r["violated_critical_requirements"] for r in rows) affected = sum(r["has_assessed_critical_violation"] is True for r in rows) complete = all(r["declared_coverage_complete"] for r in rows) return dict(declared_artifacts=len(rows), reviewed_designation_artifacts=sum(r["designation_state"] == "reviewed" for r in rows), reviewed_empty_designation_artifacts=sum(r["designation_state"] == "reviewed" and r["designated_requirements"] == 0 for r in rows), unfinished_designation_artifacts=sum(r["designation_state"] == "unfinished" for r in rows), unavailable_designation_artifacts=sum(r["designation_state"] == "unavailable" for r in rows), provided_critical_requirement_instances=known, assessed_critical_requirements=assessed, violated_critical_requirements=violated, missing_judgments=known-assessed, judgment_coverage=dict(numerator=assessed, denominator=known, rate=assessed/known if known else None), artifacts_with_assessed_critical_violation=affected, requirements_with_assessed_invented_restriction=sum(r["requirements_with_assessed_invented_restriction"] for r in rows), artifacts_with_assessed_invented_restriction=sum(r["has_assessed_invented_restriction"] is True for r in rows), declared_coverage_complete=complete, complete_rates_withheld_for_missing_coverage=not complete, critical_requirement_violation_rate=violated/assessed if complete and assessed else None, artifact_violation_rate=dict(numerator=affected, denominator=len(rows), rate=affected/len(rows) if complete and rows else None), artifacts=rows) def agreement_group(artifacts, a, b): cells = [(r["artifact_id"], q["requirement_id"]) for r in artifacts for q in r["requirements"]] common = [c for c in cells if a.get(c, {}).get("violation") is not None and b.get(c, {}).get("violation") is not None] return dict(jointly_assessed_requirement_denominator=len(common), provided_instances_missing_joint_judgments=len(cells)-len(common), unfinished_designation_artifacts=sum(r["designation_state"] == "unfinished" for r in artifacts), unavailable_designation_artifacts=sum(r["designation_state"] == "unavailable" for r in artifacts), dimensions={field: dict(agreeing_judgments=sum(a[c][field] == b[c][field] for c in common), raw_agreement=sum(a[c][field] == b[c][field] for c in common)/len(common) if common else None) for field in ("violation", "invented_restriction")}) def summarize(key_path, form_paths, registry_path=None): key, key_hash, design, artifacts, expectations, specs = load_key(key_path, registry_path) series, sources = load_ratings(form_paths, key, artifacts, expectations, specs) values = list(artifacts.values()) conditions = sorted({r["condition_id"] for r in values}) def grouped(function): return dict(overall=function(values), by_condition={c: function([r for r in values if r["condition_id"] == c]) for c in conditions}) raters = list(key["independent_rater_aliases"].values()) return dict(critical_report_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, coordinator_key_sha256=key_hash, source_manifest_sha256=design["manifest_hash"], registry_signature_sha256=key["registry_signature_sha256"], archives=design["archives"], unselected_archived_run_ids=design["unselected_archived_run_ids"], scope="human-designated public requirements, shared by task across selected conditions and repetitions; no criticality inference", rate_policy="violated / nonnull assessed critical requirement instances; missing designation/review/judgments withhold complete rates; separate from primary scoring", independent={r: grouped(lambda rows: summarize_group(rows, series[r])) for r in raters}, adjudicated=grouped(lambda rows: summarize_group(rows, series[key["adjudicator_alias"]])), independent_rater_agreement=grouped(lambda rows: agreement_group(rows, series[raters[0]], series[raters[1]])), assessment_sources=sources) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="command", required=True) prepare = sub.add_parser("build") prepare.add_argument("--manifest", type=Path, required=True) prepare.add_argument("--rater-output", type=Path, required=True) prepare.add_argument("--coordinator-output", type=Path, required=True) prepare.add_argument("--seed", type=int, default=0) report = sub.add_parser("summarize") report.add_argument("--key", type=Path, required=True) report.add_argument("--forms", type=Path, nargs="*", default=[]) for child in (prepare, report): child.add_argument("--registry", type=Path) args = parser.parse_args(argv) try: result = (build(args.manifest, args.rater_output, args.coordinator_output, args.registry, args.seed) if args.command == "build" else summarize(args.key, args.forms, args.registry)) except (sealing.ValidationError, observation.AssemblyError, sealing.account_resources.LedgerError, OSError, KeyError, TypeError, ValueError, ArithmeticError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())