#!/usr/bin/env python3 """Offline human coding of adjacent delivered-artifact transitions. Untested development tooling; identical text never generates a repetition judgment. """ import argparse from datetime import datetime, timezone import json from pathlib import Path import random import re import sys import uuid BASE = Path(__file__).resolve().parent sys.path.insert(0, str(BASE.parent/"critical-violations")) import assess_critical as shared archived, sealing, packets, scoring = shared.archived, shared.sealing, shared.packets, shared.scoring NOTICE = "Development repeated-work tooling, not empirical evidence; defaults unfrozen; human semantic judgments only." IDENTITY = ("trajectory_id", "transition_id", "task_id", "task_version", "rubric_version", "family_id", "source_packet_sha256", "before", "after", "instructions", "input_complete", "missing_inputs") RATING_FIELDS = (*IDENTITY, "repeated_work", "reason", "reviewed_scope", "completed_work_evidence", "redone_work_evidence", "task_reason_evidence", "completed_at_utc", "independent_judgments", "resolution_note") INSTRUCTIONS = """# Human repeated-work coding Untested development tooling; defaults unfrozen. Follow the local chronological order within each opaque trajectory. The unit is one adjacent delivered-artifact transition. Repeated work means redoing work already completed before this transition without a task-supported reason. Set repeated_work to integer 1 or 0; leave unresolved or input-incomplete cases null. Identical text, retention of a correct answer, or a submitted instruction alone does not prove work was redone. Necessary rechecking, repair, changed assumptions, or task-required restatement may justify returning to earlier work; explain against the fixed public packet. Use the unchanged artifacts, explicit submitted-worker instructions and public packet only. Explain completed work, apparent redo, and presence/absence of a task-supported reason. Record reviewed scope. Positive codes require exact earlier artifact and after-artifact spans; these establish locations, not semantic proof. Evidence uses Unicode code-point [start,end) offsets in UTF-8-decoded text, including CR/LF/BOM. Do not infer hidden computation or progress from prose alone. Rate independently and lock your form before sharing. A third human adjudicates both locked judgments in a separate form; agreement is never auto-copied. Never feed these assessments into adaptation. Do not open coordinator mappings, memory, usage logs, traces or prior quality scores. Artifacts and instructions are copied unchanged: both can disclose methods intrinsically. Opaque IDs cannot remove that information, and directory separation does not enforce confidentiality. """ def integer(value, minimum, label): sealing.require(type(value) is int and value >= minimum, label+": invalid integer") return value def load_selection(manifest_path, registry_path=None, source_parent=None): manifest_path = Path(manifest_path).resolve() manifest, content = shared.read_json(manifest_path, "repeated-work selection") parent = source_parent or manifest_path.parent sealing.keys(manifest, ("repeated_manifest_version", "assessment_id", "independent_rater_ids", "adjudicator_id", "archives", "runs"), "repeated-work selection") sealing.require(manifest["repeated_manifest_version"] == "1.0", "incompatible selection version") sealing.text(manifest["assessment_id"], "assessment ID") raters = manifest["independent_rater_ids"] sealing.strings(raters, "independent raters") sealing.require(len(raters) == len(set(raters)) == 2, "declare two distinct independent raters") sealing.require(sealing.text(manifest["adjudicator_id"], "adjudicator") not in raters, "adjudicator must be distinct") roots = {} for row in sealing.array(manifest["archives"], "archives"): sealing.keys(row, ("archive_id", "path"), "archive descriptor") aid = sealing.text(row["archive_id"], "archive alias") sealing.require(aid not in roots, "duplicate archive alias") roots[aid] = (parent/sealing.text(row["path"], "archive path")).resolve() collection = archived.summarize_archives(list(roots.values()), registry_path) registry_path = Path(registry_path or sealing.KIT/"task-registry.json").resolve() registry, _ = shared.read_json(registry_path, "registry") tasks, _, _ = sealing.registry_data(registry, registry_path.parent) inventories, provenance = {}, {p["archive"]: p for p in collection["archives"]} for aid, root in roots.items(): inventories[aid], _ = shared.read_json(archived.inside(root, "archive-inventory.json"), "sealed inventory") selected, seen_runs, transition_ids = [], set(), set() for run in sealing.array(manifest["runs"], "selected trajectories"): sealing.keys(run, ("archive_id", "seal_record_id", "run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id", "trajectory_status", "trajectory_note", "declared_delivery_count", "deliveries", "transitions"), "selected trajectory") rid, aid = sealing.text(run["run_id"], "run ID"), run["archive_id"] sealing.require(aid in roots and rid not in seen_runs, "unknown archive or duplicate selected run") seen_runs.add(rid) inventory = inventories[aid] source_runs = {r["run_id"]: r for r in inventory["runs"]} sealing.require(rid in source_runs, "unknown sealed run") original = source_runs[rid] seal = provenance[str(roots[aid])] sealing.require(run["seal_record_id"] == seal["seal_record_id"] and all(run[k] == original[k] for k in ("condition_id", "task_id", "task_version", "rubric_version", "repetition_id")), "run identity/task/version mismatch") state = run["trajectory_status"] sealing.require(state in ("reviewed", "unfinished", "unavailable"), "invalid trajectory review status") sealing.text(run["trajectory_note"], "trajectory scope/completeness note") count = run["declared_delivery_count"] deliveries, transitions = sealing.array(run["deliveries"], "deliveries"), sealing.array(run["transitions"], "transitions") if state == "unavailable": sealing.require(count is None and not deliveries and not transitions, "unavailable trajectory requires unknown count and empty records") else: integer(count, 0, "declared delivery count") records = {r["record_id"]: r for r in inventory["records"]} def snapshot(descriptor, kind): sealing.keys(descriptor, ("record_id", "sha256"), kind+" descriptor") record_id = sealing.text(descriptor["record_id"], "record ID") sealing.require(record_id in original["record_ids"] and records[record_id]["kind"] == kind, "record kind/run identity mismatch") record = records[record_id] sealing.require(descriptor["sha256"] == record["sha256"], "record source hash mismatch") data = packets.hashed_bytes(archived.inside(roots[aid], record["path"]), record["sha256"]) data.decode("utf-8") return data ordered, used_records, previous_sequence, previous_time = {}, set(), 0, None seal_time = sealing.utc(seal["sealed_at_utc"], "archive seal time") run_end = sealing.utc_optional(original["ended_at_utc"], "run end") for delivery in deliveries: sealing.keys(delivery, ("sequence", "delivered_at_utc", "artifact"), "delivered artifact") seq = integer(delivery["sequence"], 1, "delivery sequence") stamp = sealing.utc_optional(delivery["delivered_at_utc"], "delivery time") sealing.require(count is not None and previous_sequence < seq <= count and (stamp is None or previous_time is None or previous_time < stamp), "duplicate/reversed delivery chronology or out-of-range sequence") sealing.require(stamp is None or (stamp <= seal_time and (run_end is None or stamp <= run_end)), "delivery occurs after sealing/run end") data = snapshot(delivery["artifact"], "artifact") record_id = delivery["artifact"]["record_id"] sealing.require(record_id not in used_records, "duplicate delivered artifact record identity") used_records.add(record_id) ordered[seq] = dict(source=delivery, content=data) previous_sequence = seq if stamp is not None: previous_time = stamp attempts = {a["attempt_id"]: a for a in original["attempts"]} edges, previous_edge, used_events, previous_event_time = {}, 0, set(), None for edge in transitions: sealing.keys(edge, ("transition_id", "earlier_sequence", "later_sequence", "instruction_status", "instruction_note", "instructions"), "declared transition") eid = sealing.text(edge["transition_id"], "source transition ID") earlier, later = integer(edge["earlier_sequence"], 1, "earlier sequence"), integer(edge["later_sequence"], 1, "later sequence") sealing.require(eid not in transition_ids and earlier not in edges, "duplicate transition identity/cell") sealing.require(count is not None and previous_edge < earlier and later == earlier+1 and later <= count, "reversed/nonadjacent transition chronology or out-of-range endpoints") transition_ids.add(eid) previous_edge = earlier status = edge["instruction_status"] sealing.require(status in ("complete", "unfinished", "unavailable"), "invalid instruction coverage status") instructions = sealing.array(edge["instructions"], "submitted instructions") if status == "complete": sealing.require(instructions and edge["instruction_note"] is None, "complete instructions require records and a null issue note") else: sealing.text(edge["instruction_note"], "missing instruction explanation") sealing.require(status != "unavailable" or not instructions, "unavailable instructions cannot contain records") loaded = [] for instruction in instructions: sealing.keys(instruction, ("record_id", "sha256", "attempt_id", "submitted_at_utc"), "submitted worker instruction") descriptor = {k: instruction[k] for k in ("record_id", "sha256")} data = snapshot(descriptor, "submitted_prompt") attempt_id = instruction["attempt_id"] sealing.require(attempt_id in attempts, "unknown instruction attempt") attempt = attempts[attempt_id] sealing.require(attempt["phase"] == "adaptation" and attempt["role"] == "worker" and instruction["record_id"] in attempt["record_ids"], "instruction must reference its sealed adaptation-worker attempt") event = (instruction["record_id"], attempt_id) sealing.require(event not in used_events, "duplicate submitted instruction event") used_events.add(event) stamp = sealing.utc_optional(instruction["submitted_at_utc"], "instruction submission") sealing.require(stamp is None or previous_event_time is None or previous_event_time < stamp, "reversed submitted instruction chronology") sealing.require(stamp is None or (stamp <= seal_time and (run_end is None or stamp <= run_end)), "instruction occurs after sealing/run end") for sequence, known in ordered.items(): clock = sealing.utc_optional(known["source"]["delivered_at_utc"], "delivery time") if clock is not None and stamp is not None: if sequence <= earlier: sealing.require(clock < stamp, "instruction must follow earlier delivery") elif sequence >= later: sealing.require(stamp <= clock, "instruction must precede later delivery") start, end = (sealing.utc_optional(attempt[k], k) for k in ("started_at_utc", "ended_at_utc")) sealing.require(stamp is None or ((start is None or start <= stamp) and (end is None or stamp <= end)), "submission outside declared worker attempt times") if stamp is not None: previous_event_time = stamp loaded.append(dict(source=instruction, content=data)) edges[earlier] = dict(source=edge, instructions=loaded) public = tasks[run["task_id"]]["public_packet"] public_bytes = packets.hashed_bytes(registry_path.parent/public["path"], public["sha256"]) public_bytes.decode("utf-8") selected.append(dict(source=run, deliveries=ordered, edges=edges, source_packet=public_bytes, source_packet_sha256=public["sha256"], family_id=tasks[run["task_id"]]["family_id"], sealed_at_utc=seal["sealed_at_utc"])) sealing.require(selected, "declare at least one trajectory") return dict(manifest=manifest, manifest_bytes=content, manifest_hash=sealing.sha(content), registry=registry, runs=sorted(selected, key=lambda r: r["source"]["run_id"]), archives=collection["archives"], unselected_archived_run_ids=sorted(r["run_id"] for r in collection["runs"] if r["run_id"] not in seen_runs)) def render(design, seed, manifest_parent, opaque): """Deterministic packet/key construction given explicit opaque IDs; no writes.""" raters = {r: opaque("rater") for r in design["manifest"]["independent_rater_ids"]} adjudicator = opaque("rater") key = dict(repeated_key_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, batch_id=opaque("batch"), seed=seed, manifest_parent=str(manifest_parent), source_manifest_sha256=design["manifest_hash"], registry_signature_sha256=sealing.sha(archived.registry_signature(design["registry"])), independent_rater_aliases=raters, adjudicator_alias=adjudicator, trajectories=[], forms=[]) files = {} for item in design["runs"]: source, trajectory_id = item["source"], opaque("trajectory") count = source["declared_delivery_count"] row = dict(trajectory_id=trajectory_id, source_run_id=source["run_id"], condition_id=source["condition_id"], repetition_id=source["repetition_id"], task_id=source["task_id"], task_version=source["task_version"], rubric_version=source["rubric_version"], family_id=item["family_id"], source_packet_sha256=item["source_packet_sha256"], sealed_at_utc=item["sealed_at_utc"], trajectory_status=source["trajectory_status"], declared_delivery_count=count, expected_transition_count=None if count is None else max(count-1, 0), missing_delivery_sequences=[] if count is None else [s for s in range(1, count+1) if s not in item["deliveries"]], artifacts=[], transitions=[]) files["public/"+item["source_packet_sha256"]+".txt"] = item["source_packet"] by_sequence = {} for seq, delivery in item["deliveries"].items(): aid = opaque("artifact") descriptor = dict(artifact_id=aid, sha256=delivery["source"]["artifact"]["sha256"], sequence=seq) by_sequence[seq] = descriptor row["artifacts"].append(dict(descriptor, source_record_id=delivery["source"]["artifact"]["record_id"])) files["artifacts/"+aid+".txt"] = delivery["content"] for earlier in range(1, (count or 0)): edge = item["edges"].get(earlier) instructions, origins = [], [] if edge is not None: for instruction in edge["instructions"]: iid = opaque("instruction") instructions.append(dict(instruction_id=iid, sha256=instruction["source"]["sha256"])) origins.append(dict(instruction_id=iid, record_id=instruction["source"]["record_id"], attempt_id=instruction["source"]["attempt_id"])) files["instructions/"+iid+".txt"] = instruction["content"] missing = [] if edge is None: missing.append("transition_declaration") if earlier not in by_sequence: missing.append("before_artifact") if earlier+1 not in by_sequence: missing.append("after_artifact") if edge is None or edge["source"]["instruction_status"] != "complete": missing.append("submitted_instructions") transition = dict(trajectory_id=trajectory_id, transition_id=opaque("transition"), **{k: row[k] for k in ("task_id", "task_version", "rubric_version", "family_id", "source_packet_sha256")}, before=by_sequence.get(earlier), after=by_sequence.get(earlier+1), instructions=instructions, input_complete=not missing, missing_inputs=missing, earlier_sequence=earlier, later_sequence=earlier+1, source_transition_id=None if edge is None else edge["source"]["transition_id"], instruction_sources=origins) row["transitions"].append(transition) key["trajectories"].append(row) rng, bundles = random.Random(seed), [] for kind, assessor in [("independent", r) for r in raters.values()]+[("adjudicated", adjudicator)]: ordered = list(key["trajectories"]) rng.shuffle(ordered) # Preserve chronology inside every run block. form = dict(repeated_form_version="1.0", notice=NOTICE, defaults_unfrozen=True, batch_id=key["batch_id"], form_id=opaque("form"), assessor_id=assessor, series_kind=kind, locked_at_utc=None, ratings=[]) public_runs = [] for row in ordered: transitions = [] for transition in row["transitions"]: identity = {k: transition[k] for k in IDENTITY} form["ratings"].append(dict(identity, repeated_work=None, reason=None, reviewed_scope=None, completed_work_evidence=[], redone_work_evidence=[], task_reason_evidence=[], completed_at_utc=None, independent_judgments=None, resolution_note=None)) transitions.append(dict(identity, earlier_sequence=transition["earlier_sequence"], later_sequence=transition["later_sequence"], instruction_paths=["instructions/"+i["instruction_id"]+".txt" for i in transition["instructions"]])) public_runs.append(dict({k: row[k] for k in ("trajectory_id", "task_id", "task_version", "rubric_version", "family_id", "source_packet_sha256", "trajectory_status", "declared_delivery_count", "expected_transition_count", "missing_delivery_sequences")}, public_packet_path="public/"+row["source_packet_sha256"]+".txt", artifacts=[dict({k: a[k] for k in ("artifact_id", "sha256", "sequence")}, path="artifacts/"+a["artifact_id"]+".txt") for a in row["artifacts"]], transitions=transitions)) key["forms"].append(dict(assessor_id=assessor, form_id=form["form_id"], series_kind=kind, trajectory_order=[r["trajectory_id"] for r in ordered])) index = dict(packet_version="1.0", notice=NOTICE, defaults_unfrozen=True, batch_id=key["batch_id"], assessor_id=assessor, trajectories=public_runs) bundles.append((assessor, form, index)) return key, files, bundles def build(manifest_path, rater_dir, coordinator_dir, registry_path=None, seed=0): sealing.require(type(seed) is int, "presentation seed must be integer") rater_dir, coordinator_dir = packets.output_path(rater_dir), packets.output_path(coordinator_dir) sealing.require(packets.disjoint(rater_dir, coordinator_dir), "rater and coordinator outputs must be disjoint") design = load_selection(manifest_path, registry_path) for output in (rater_dir, coordinator_dir): sealing.require(all(output != Path(p["archive"]) and Path(p["archive"]) not in output.parents for p in design["archives"]), "outputs must stay outside sealed archives") issued, used = [], set() def opaque(prefix): value = prefix+"-"+uuid.uuid4().hex sealing.require(value not in used, "opaque ID collision") used.add(value) issued.append(dict(kind=prefix, value=value)) return value key, files, bundles = render(design, seed, Path(manifest_path).resolve().parent, opaque) key["issued_ids"] = issued rater_dir.mkdir() coordinator_dir.mkdir() for assessor, form, index in bundles: root = rater_dir/assessor root.mkdir() for relative, content in files.items(): packets.write_bytes(root/relative, content) packets.write_json(root/"rating-form.json", form) packets.write_json(root/"packet-index.json", index) packets.write_bytes(root/"INSTRUCTIONS.md", INSTRUCTIONS.encode("utf-8")) packets.write_bytes(coordinator_dir/"source-manifest.json", design["manifest_bytes"]) packets.write_json(coordinator_dir/"repeated-key.json", key) # Last completed-build record. return dict(notice=NOTICE, defaults_unfrozen=True, trajectory_count=len(key["trajectories"]), blank_form_count=3) def load_key(key_path, registry_path=None): key_path = Path(key_path).resolve() key, key_bytes = shared.read_json(key_path, "repeated-work key") sealing.keys(key, ("repeated_key_version", "notice", "coordinator_only", "defaults_unfrozen", "batch_id", "seed", "manifest_parent", "source_manifest_sha256", "registry_signature_sha256", "independent_rater_aliases", "adjudicator_alias", "trajectories", "forms", "issued_ids"), "repeated-work key") sealing.require(key["repeated_key_version"] == "1.0", "incompatible key version") sealing.require(type(key["seed"]) is int, "invalid presentation seed") parent = Path(sealing.text(key["manifest_parent"], "original manifest parent")) sealing.require(parent.is_absolute(), "manifest parent must be absolute") manifest_path = archived.inside(key_path.parent, "source-manifest.json") packets.hashed_bytes(manifest_path, key["source_manifest_sha256"]) design = load_selection(manifest_path, registry_path, source_parent=parent) ids = iter(sealing.array(key["issued_ids"], "issued opaque IDs")) used = set() def replay(prefix): entry = next(ids, None) sealing.keys(entry, ("kind", "value"), "issued ID") value = entry["value"] sealing.require(entry["kind"] == prefix and isinstance(value, str) and re.fullmatch(prefix+r"-[0-9a-f]{32}", value) is not None and value not in used, "invalid/duplicate opaque identity or issue order") used.add(value) return value expected, files, _ = render(design, key["seed"], parent, replay) sealing.require(next(ids, None) is None, "extra issued opaque IDs") expected["issued_ids"] = key["issued_ids"] sealing.require(shared.same(expected, key), "key source identity/hash/coverage/version mismatch") return key, sealing.sha(key_bytes), design, files def validate_evidence(row, transition, trajectory, files): artifacts = {a["artifact_id"]: a for a in trajectory["artifacts"]} for field in ("completed_work_evidence", "redone_work_evidence"): for ref in sealing.array(row[field], field): sealing.keys(ref, ("artifact_id", "sha256", "start", "end", "text"), "artifact evidence") aid = ref["artifact_id"] sealing.require(aid in artifacts and ref["sha256"] == artifacts[aid]["sha256"], "artifact evidence identity/hash mismatch or wrong trajectory") if field == "completed_work_evidence": sealing.require(artifacts[aid]["sequence"] <= transition["earlier_sequence"], "completed-work evidence cannot cite future artifacts") else: sealing.require(transition["after"] is not None and aid == transition["after"]["artifact_id"], "redo evidence must cite designated after artifact") shared.span({k: ref[k] for k in ("start", "end", "text")}, files["artifacts/"+aid+".txt"].decode("utf-8"), "artifact evidence") instructions = {i["instruction_id"]: i for i in transition["instructions"]} for ref in sealing.array(row["task_reason_evidence"], "task-reason evidence"): sealing.keys(ref, ("channel", "document_id", "sha256", "start", "end", "text"), "task-reason evidence") if ref["channel"] == "public_packet": sealing.require(ref["document_id"] == trajectory["task_id"] and ref["sha256"] == trajectory["source_packet_sha256"], "public packet evidence identity/hash mismatch") path = "public/"+trajectory["source_packet_sha256"]+".txt" else: sealing.require(ref["channel"] == "instruction" and ref["document_id"] in instructions, "instruction evidence is not supplied for this transition") sealing.require(ref["sha256"] == instructions[ref["document_id"]]["sha256"], "instruction evidence hash mismatch") path = "instructions/"+ref["document_id"]+".txt" shared.span({k: ref[k] for k in ("start", "end", "text")}, files[path].decode("utf-8"), "task-reason evidence") def load_ratings(form_paths, key, files): specs = {s["assessor_id"]: s for s in key["forms"]} transitions = {t["transition_id"]: (t, run) for run in key["trajectories"] for t in run["transitions"]} series, locks, seen, sources = {s: {} for s in specs}, {}, set(), [] now = datetime.now(timezone.utc) for path in form_paths: form, content = shared.read_json(path, "human repeated-work form") sealing.keys(form, ("repeated_form_version", "notice", "defaults_unfrozen", "batch_id", "form_id", "assessor_id", "series_kind", "locked_at_utc", "ratings"), "repeated-work form") sealing.require(form["repeated_form_version"] == "1.0", "incompatible form version") assessor = form["assessor_id"] sealing.require(assessor in specs and assessor not in seen, "unknown/duplicate assessment series") spec = specs[assessor] sealing.require(form["notice"] == NOTICE and form["defaults_unfrozen"] is True and form["batch_id"] == key["batch_id"] and all(form[k] == spec[k] for k in ("form_id", "series_kind")), "form batch/series identity or development label mismatch") lock = None if form["locked_at_utc"] is None else sealing.utc(form["locked_at_utc"], "form lock") sealing.require(lock is None or lock <= now, "future form lock") locks[assessor] = lock seen.add(assessor) sources.append(dict(path=str(Path(path).resolve()), sha256=sealing.sha(content), assessor_id=assessor, series_kind=form["series_kind"])) for row in sealing.array(form["ratings"], "transition ratings"): sealing.keys(row, RATING_FIELDS, "transition rating") tid = row["transition_id"] sealing.require(tid in transitions and tid not in series[assessor], "unknown/duplicate assessed transition") transition, trajectory = transitions[tid] sealing.require(shared.same({k: row[k] for k in IDENTITY}, {k: transition[k] for k in IDENTITY}), "transition/artifact/instruction identity or hash mismatch") value = scoring.binary(row["repeated_work"], "repeated-work judgment") validate_evidence(row, transition, trajectory, files) if value is None: sealing.require(row["completed_at_utc"] is None and row["independent_judgments"] is None and row["resolution_note"] is None, "missing judgment marked completed/resolved") for field in ("reason", "reviewed_scope"): sealing.require(row[field] is None or isinstance(row[field], str), "invalid pending assessment note") else: sealing.require(transition["input_complete"], "cannot judge an input-incomplete transition") sealing.text(row["reason"], "human repetition rationale") sealing.text(row["reviewed_scope"], "human review scope") if value == 1: sealing.require(row["completed_work_evidence"] and row["redone_work_evidence"], "positive judgment requires earlier completed-work and after redo evidence") stamp = sealing.utc(row["completed_at_utc"], "assessment completion") sealing.require(lock is not None and sealing.utc(trajectory["sealed_at_utc"], "archive seal") <= stamp <= lock, "judgment must follow sealing and precede form lock") if form["series_kind"] == "independent": sealing.require(row["independent_judgments"] is None and row["resolution_note"] is None, "independent form contains adjudication fields") series[assessor][tid] = row raters = list(key["independent_rater_aliases"].values()) for tid, row in series[key["adjudicator_alias"]].items(): if row["repeated_work"] is None: continue sealing.keys(row["independent_judgments"], raters, "adjudication basis") sealing.text(row["resolution_note"], "human resolution note") for rater in raters: independent = series[rater].get(tid) sealing.require(independent is not None and independent["repeated_work"] is not None, "adjudication requires both independent judgments") sealing.require(shared.same(row["independent_judgments"][rater], independent["repeated_work"]), "stale adjudication basis") sealing.require(sealing.utc(row["completed_at_utc"], "adjudication time") >= locks[rater], "adjudication predates independent lock") return series, sources def summarize_group(trajectories, ratings): rows = [] for run in trajectories: expected = run["expected_transition_count"] eligible = [t for t in run["transitions"] if t["input_complete"]] assessed = [ratings[t["transition_id"]] for t in eligible if t["transition_id"] in ratings and ratings[t["transition_id"]]["repeated_work"] is not None] repeated = sum(r["repeated_work"] for r in assessed) complete = run["trajectory_status"] == "reviewed" and not run["missing_delivery_sequences"] and expected == len(assessed) missing = [t["transition_id"] for t in run["transitions"] if t["transition_id"] not in ratings or ratings[t["transition_id"]]["repeated_work"] is None] rows.append(dict(trajectory_id=run["trajectory_id"], run_id=run["source_run_id"], trajectory_status=run["trajectory_status"], expected_transitions=expected, available_transitions=len(eligible), assessed_transitions=len(assessed), repeated_work_transitions=repeated, missing_transition_inputs=None if expected is None else expected-len(eligible), missing_judgments=len(eligible)-len(assessed), missing_transition_coverage=None if expected is None else expected-len(assessed), missing_coverage_transition_ids=missing, missing_delivery_sequences=run["missing_delivery_sequences"], declared_coverage_complete=complete, affected=True if repeated else (False if complete else None), repeated_work_rate=repeated/len(assessed) if complete and assessed else None)) expected = sum(r["expected_transitions"] for r in rows if r["expected_transitions"] is not None) available = sum(r["available_transitions"] for r in rows) assessed = sum(r["assessed_transitions"] for r in rows) repeated = sum(r["repeated_work_transitions"] for r in rows) affected = sum(r["affected"] is True for r in rows) complete = all(r["declared_coverage_complete"] for r in rows) return dict(declared_runs=len(rows), reviewed_runs=sum(r["trajectory_status"] == "reviewed" for r in rows), unfinished_trajectory_runs=sum(r["trajectory_status"] == "unfinished" for r in rows), unavailable_trajectory_runs=sum(r["trajectory_status"] == "unavailable" for r in rows), reviewed_no_transition_runs=sum(r["trajectory_status"] == "reviewed" and r["expected_transitions"] == 0 for r in rows), expected_known_transitions=expected, available_transitions=available, assessed_transitions=assessed, repeated_work_transitions=repeated, missing_transition_inputs=expected-available, missing_judgments=available-assessed, missing_transition_coverage=expected-assessed, known_transition_coverage=dict(numerator=assessed, denominator=expected, rate=assessed/expected if expected else None), confirmed_affected_runs=affected, declared_coverage_complete=complete, complete_rates_withheld_for_missing_coverage=not complete, repeated_work_rate=repeated/assessed if complete and assessed else None, affected_run_rate=dict(numerator=affected, denominator=len(rows), rate=affected/len(rows) if complete and rows else None), runs=rows) def agreement_group(trajectories, a, b): transitions = [t for run in trajectories for t in run["transitions"]] eligible = [t["transition_id"] for t in transitions if t["input_complete"]] common = [tid for tid in eligible if a.get(tid, {}).get("repeated_work") is not None and b.get(tid, {}).get("repeated_work") is not None] agreeing = sum(a[tid]["repeated_work"] == b[tid]["repeated_work"] for tid in common) return dict(agreeing_transitions=agreeing, jointly_assessed_transition_denominator=len(common), available_transitions_missing_joint_judgments=len(eligible)-len(common), input_incomplete_transitions=len(transitions)-len(eligible), unavailable_trajectory_runs=sum(r["trajectory_status"] == "unavailable" for r in trajectories), unfinished_trajectory_runs=sum(r["trajectory_status"] == "unfinished" for r in trajectories), raw_agreement=agreeing/len(common) if common else None) def summarize(key_path, form_paths, registry_path=None): key, key_hash, design, files = load_key(key_path, registry_path) series, sources = load_ratings(form_paths, key, files) trajectories = key["trajectories"] conditions = sorted({r["condition_id"] for r in trajectories}) def grouped(function): return dict(overall=function(trajectories), by_condition={c: function([r for r in trajectories if r["condition_id"] == c]) for c in conditions}) raters = list(key["independent_rater_aliases"].values()) return dict(repeated_report_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, coordinator_key_sha256=key_hash, source_manifest_sha256=design["manifest_hash"], registry_signature_sha256=key["registry_signature_sha256"], archives=design["archives"], unselected_archived_run_ids=design["unselected_archived_run_ids"], unit="one adjacent declared delivered-artifact transition; one binary human code regardless of number of repeated subactivities", rate_policy="positive repeated-work codes / nonnull assessed transitions; no imputation, byte-similarity rule, or progress score; complete rates require reviewed trajectories and all declared inputs/judgments", independent={r: grouped(lambda rows: summarize_group(rows, series[r])) for r in raters}, adjudicated=grouped(lambda rows: summarize_group(rows, series[key["adjudicator_alias"]])), independent_rater_agreement=grouped(lambda rows: agreement_group(rows, series[raters[0]], series[raters[1]])), assessment_sources=sources) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="command", required=True) prepare = sub.add_parser("build") prepare.add_argument("--manifest", type=Path, required=True) prepare.add_argument("--rater-output", type=Path, required=True) prepare.add_argument("--coordinator-output", type=Path, required=True) prepare.add_argument("--seed", type=int, default=0) report = sub.add_parser("summarize") report.add_argument("--key", type=Path, required=True) report.add_argument("--forms", type=Path, nargs="*", default=[]) for child in (prepare, report): child.add_argument("--registry", type=Path) args = parser.parse_args(argv) try: result = (build(args.manifest, args.rater_output, args.coordinator_output, args.registry, args.seed) if args.command == "build" else summarize(args.key, args.forms, args.registry)) except (sealing.ValidationError, sealing.account_resources.LedgerError, OSError, KeyError, TypeError, ValueError, ArithmeticError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())