#!/usr/bin/env python3 """Offline human-enumerated factual-claim support packets and assessment summaries. Development tooling only. Spans are validated; factuality and support are not inferred. """ import argparse from datetime import datetime, timezone import json from pathlib import Path import random import re import sys import uuid BASE = Path(__file__).resolve().parent sys.path.insert(0, str(BASE.parent/"run-sealing")) import summarize_run_outcomes as archived sealing = archived.sealing import build_assessment_packets as packets NOTICE = "Development human claim-support tooling, not empirical evidence; defaults unfrozen; no automatic claim identification or judgments." PUBLIC_ARTIFACT = ("artifact_id", "artifact_sha256", "task_id", "task_version", "rubric_version", "family_id", "source_packet_sha256") CLAIM_IDENTITY = (*PUBLIC_ARTIFACT, "inventory_sha256", "claim_id", "start", "end", "text") JUDGMENTS = ("supported", "unsupported") RATING_FIELDS = (*CLAIM_IDENTITY, "judgment", "reason", "evidence_locations", "completed_at_utc", "independent_judgments", "resolution_note") INSTRUCTIONS = """# Human factual-claim support assessment Untested development assets; defaults remain unfrozen. Assess only the human- enumerated claims in the supplied final artifact and fixed public task/source packet. Claim offsets are zero-based Unicode code-point [start,end) positions in the exact UTF-8-decoded artifact, including CR, LF and any BOM. Bytes are unchanged. Enumeration is a human declaration, not proof that every factual claim was found. Record supported or unsupported for each claim, a reason, precise source/artifact locations, and a UTC completion time. Supported means warranted by the authorized packet (including justified derivation); unsupported means not warranted there, including contradictions. Citation syntax or existence never establishes support. Leave unresolved judgments null, with a reason if possible. An item that appears nonfactual or poorly segmented needs coordinator review, not a forced support label. Do not silently delete, merge, split, or add claims or consult outside sources. Rate independently and lock the form before sharing judgments. A separate human adjudicator reviews both locked judgments and supplies a separate decision with both original classifications and a resolution note. Agreement is not auto-copied. No ratings or inventories may be returned as adaptation feedback. Do not open coordinator keys, condition mappings, prompts, memory or resource logs. Opaque IDs cannot remove method disclosures inside artifact text. Report blinding issues rather than editing artifacts. Directory separation does not enforce confidentiality. This metric does not measure exhaustive factual-error detection. """ def read_json(path, label): content = Path(path).read_bytes() return sealing.parse_json(content, label), content def validate_inventory(content, selection, artifact_text, seal_time): inventory = sealing.parse_json(content, "human claim inventory") sealing.keys(inventory, ("claim_inventory_version", "artifact_id", "artifact_sha256", "task_id", "task_version", "rubric_version", "enumeration_status", "enumerator_id", "completed_at_utc", "note", "claims"), "claim inventory") sealing.require(inventory["claim_inventory_version"] == "1.0", "incompatible claim inventory version") sealing.require(all(inventory[k] == selection[k] for k in ("artifact_id", "artifact_sha256", "task_id", "task_version", "rubric_version")), "claim inventory artifact identity/hash/version mismatch") sealing.require(inventory["enumeration_status"] in ("reviewed", "unfinished"), "invalid enumeration status") sealing.text(inventory["enumerator_id"], "human enumerator ID") sealing.text(inventory["note"], "enumeration scope/limitations note") if inventory["enumeration_status"] == "reviewed": stamp = sealing.utc(inventory["completed_at_utc"], "inventory review time") sealing.require(seal_time <= stamp <= datetime.now(timezone.utc), "enumeration review must follow sealing and not be in the future") else: sealing.require(inventory["completed_at_utc"] is None, "unfinished inventory cannot claim completed review") seen, spans = set(), set() for claim in sealing.array(inventory["claims"], "enumerated claims"): sealing.keys(claim, ("claim_id", "start", "end", "text"), "enumerated claim") cid = sealing.text(claim["claim_id"], "human claim ID") sealing.require(cid not in seen, "duplicate claim identity within artifact") seen.add(cid) start, end = claim["start"], claim["end"] sealing.require(type(start) is int and type(end) is int and 0 <= start < end <= len(artifact_text), "invalid claim span bounds") sealing.require((start, end) not in spans, "duplicate claim span within artifact") spans.add((start, end)) sealing.text(claim["text"], "claim text") sealing.require(artifact_text[start:end] == claim["text"], "claim span does not match exact artifact text") return inventory def load_selection(manifest_path, registry_path=None, source_parent=None, inventory_copies=None): manifest_path = Path(manifest_path).resolve() manifest, manifest_bytes = read_json(manifest_path, "claim assessment selection") parent = source_parent or manifest_path.parent sealing.keys(manifest, ("claim_manifest_version", "assessment_id", "independent_rater_ids", "adjudicator_id", "archives", "artifacts"), "claim assessment selection") sealing.require(manifest["claim_manifest_version"] == "1.0", "incompatible claim selection version") sealing.text(manifest["assessment_id"], "assessment_id") raters = manifest["independent_rater_ids"] sealing.strings(raters, "independent raters") sealing.require(len(raters) == len(set(raters)) == 2, "declare two distinct independent raters") sealing.require(sealing.text(manifest["adjudicator_id"], "adjudicator") not in raters, "adjudicator must be distinct") roots = {} for descriptor in sealing.array(manifest["archives"], "archives"): sealing.keys(descriptor, ("archive_id", "path"), "archive descriptor") aid = sealing.text(descriptor["archive_id"], "archive alias") sealing.require(aid not in roots, "duplicate archive alias") roots[aid] = (parent/sealing.text(descriptor["path"], "archive path")).resolve() collection = archived.summarize_archives(list(roots.values()), registry_path) registry_path = Path(registry_path or sealing.KIT/"task-registry.json").resolve() registry, _ = read_json(registry_path, "selected registry") tasks, _, _ = sealing.registry_data(registry, registry_path.parent) endpoints, commits = {}, {} for aid, root in roots.items(): commit, _ = read_json(archived.inside(root, "packet-build-manifest.json"), "archive commit") commits[aid] = commit for run in commit["runs"]: endpoints[(aid, run["run_id"])] = run selected, seen, copies = [], set(), {} for row in sealing.array(manifest["artifacts"], "selected artifacts"): sealing.keys(row, ("archive_id", "seal_record_id", "run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id", "artifact_id", "artifact_sha256", "inventory", "unavailability_reason"), "selected artifact") rid, aid = sealing.text(row["run_id"], "run ID"), row["archive_id"] sealing.require(rid not in seen and (aid, rid) in endpoints, "duplicate selection or run lacks a sealed designated final artifact") seen.add(rid) run, commit = endpoints[(aid, rid)], commits[aid] sealing.require(row["seal_record_id"] == commit["seal_record_id"] and all(row[k] == run[k] for k in ("condition_id", "task_id", "task_version", "rubric_version", "repetition_id")), "selected run identity/version mismatch") final = run["final_artifact"] sealing.require(row["artifact_id"] == final["artifact_id"] and row["artifact_sha256"] == final["sha256"], "selected final artifact identity/hash mismatch") content = packets.hashed_bytes(archived.inside(roots[aid], final["path"]), final["sha256"]) text = content.decode("utf-8") # No newline/BOM conversion, normalization, or stripping. inventory = None if row["inventory"] is None: sealing.text(row["unavailability_reason"], "unavailable inventory explanation") else: descriptor = row["inventory"] sealing.keys(descriptor, ("path", "sha256"), "claim inventory descriptor") sealing.text(descriptor["path"], "explicit inventory path") sealing.digest(descriptor["sha256"], "inventory sha256") sealing.require(row["unavailability_reason"] is None, "available inventory has unavailability explanation") inventory_bytes = (inventory_copies[rid] if inventory_copies is not None else packets.hashed_bytes(parent/descriptor["path"], descriptor["sha256"])) sealing.require(sealing.sha(inventory_bytes) == descriptor["sha256"], "claim inventory source hash mismatch") inventory = validate_inventory(inventory_bytes, row, text, sealing.utc(commit["sealed_at_utc"], "seal time")) copies[rid] = inventory_bytes reference = tasks[row["task_id"]]["public_packet"] public_bytes = packets.hashed_bytes(registry_path.parent/reference["path"], reference["sha256"]) selected.append(dict(source=row, content=content, inventory=inventory, source_packet=public_bytes, source_packet_sha256=reference["sha256"], family_id=tasks[row["task_id"]]["family_id"], sealed_at_utc=commit["sealed_at_utc"])) sealing.require(selected, "declare at least one final artifact") if inventory_copies is not None: sealing.require(set(inventory_copies) == set(copies), "copied inventory coverage mismatch") return dict(manifest=manifest, manifest_bytes=manifest_bytes, manifest_hash=sealing.sha(manifest_bytes), registry=registry, selected=sorted(selected, key=lambda item: item["source"]["run_id"]), inventory_copies=copies, archives=collection["archives"], unselected_archived_run_ids=sorted(r["run_id"] for r in collection["runs"] if r["run_id"] not in seen)) def public_inventory(item, artifact_id, opaque): inventory = item["inventory"] state = "unavailable" if inventory is None else inventory["enumeration_status"] return dict(inventory_version="1.0", notice=NOTICE, defaults_unfrozen=True, artifact_id=artifact_id, artifact_sha256=item["source"]["artifact_sha256"], inventory_state=state, claims=[] if inventory is None else [dict(claim_id=opaque("claim"), start=c["start"], end=c["end"], text=c["text"]) for c in inventory["claims"]]) def build(manifest_path, rater_dir, coordinator_dir, registry_path=None, seed=0): sealing.require(type(seed) is int, "seed must be an integer") rater_dir, coordinator_dir = packets.output_path(rater_dir), packets.output_path(coordinator_dir) sealing.require(packets.disjoint(rater_dir, coordinator_dir), "coordinator and rater outputs must be disjoint") design = load_selection(manifest_path, registry_path) for output in (rater_dir, coordinator_dir): sealing.require(all(output != Path(p["archive"]) and Path(p["archive"]) not in output.parents for p in design["archives"]), "outputs must stay outside sealed archives") issued = set() def opaque(prefix): value = prefix+"-"+uuid.uuid4().hex sealing.require(value not in issued, "opaque ID collision") issued.add(value) return value raters = {r: opaque("rater") for r in design["manifest"]["independent_rater_ids"]} adjudicator = opaque("rater") key = dict(claim_key_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, batch_id=opaque("batch"), seed=seed, manifest_parent=str(Path(manifest_path).resolve().parent), source_manifest_sha256=design["manifest_hash"], registry_signature_sha256=sealing.sha(archived.registry_signature(design["registry"])), independent_rater_aliases=raters, adjudicator_alias=adjudicator, artifacts=[], inventory_copies=[], forms=[]) artifacts, inventories, references = {}, {}, {} for item in design["selected"]: source = item["source"] aid = opaque("artifact") inventory = public_inventory(item, aid, opaque) inventory_bytes = sealing.json_bytes(inventory) row = dict(artifact_id=aid, artifact_sha256=source["artifact_sha256"], task_id=source["task_id"], task_version=source["task_version"], rubric_version=source["rubric_version"], family_id=item["family_id"], source_packet_sha256=item["source_packet_sha256"], inventory_sha256=sealing.sha(inventory_bytes), source_run_id=source["run_id"], condition_id=source["condition_id"], repetition_id=source["repetition_id"], sealed_at_utc=item["sealed_at_utc"], inventory_state=inventory["inventory_state"], claims=[], rating_not_before_utc=(item["inventory"]["completed_at_utc"] if inventory["inventory_state"] == "reviewed" else item["sealed_at_utc"])) for public, original in zip(inventory["claims"], [] if item["inventory"] is None else item["inventory"]["claims"]): row["claims"].append(dict(public, source_claim_id=original["claim_id"])) key["artifacts"].append(row) artifacts[aid], inventories[aid] = item["content"], inventory_bytes references[source["task_id"]] = item["source_packet"] rng = random.Random(seed) bundles = [] for kind, assessor in [("independent", r) for r in raters.values()]+[("adjudicated", adjudicator)]: ordered = list(key["artifacts"]) rng.shuffle(ordered) form = dict(claim_form_version="1.0", notice=NOTICE, defaults_unfrozen=True, batch_id=key["batch_id"], form_id=opaque("form"), assessor_id=assessor, series_kind=kind, locked_at_utc=None, ratings=[]) for artifact in ordered: for claim in artifact["claims"]: identity = {k: artifact[k] for k in (*PUBLIC_ARTIFACT, "inventory_sha256")} identity.update({k: claim[k] for k in ("claim_id", "start", "end", "text")}) form["ratings"].append(dict(identity, judgment=None, reason=None, evidence_locations=[], completed_at_utc=None, independent_judgments=None, resolution_note=None)) key["forms"].append(dict(assessor_id=assessor, form_id=form["form_id"], series_kind=kind, artifact_order=[a["artifact_id"] for a in ordered])) index = dict(packet_version="1.0", notice=NOTICE, defaults_unfrozen=True, batch_id=key["batch_id"], assessor_id=assessor, artifacts=[dict({k: a[k] for k in (*PUBLIC_ARTIFACT, "inventory_sha256", "inventory_state")}, artifact_path="artifacts/"+a["artifact_id"]+".txt", inventory_path="claims/"+a["artifact_id"]+".json", source_packet_path="public/"+a["task_id"]+".txt") for a in ordered]) bundles.append((assessor, form, index)) for index, (rid, content) in enumerate(sorted(design["inventory_copies"].items())): key["inventory_copies"].append(dict(run_id=rid, path=f"inventories/{index}.json", sha256=sealing.sha(content))) rater_dir.mkdir() coordinator_dir.mkdir() for assessor, form, index in bundles: root = rater_dir/assessor root.mkdir() packets.write_json(root/"rating-form.json", form) packets.write_json(root/"packet-index.json", index) packets.write_bytes(root/"INSTRUCTIONS.md", INSTRUCTIONS.encode("utf-8")) for aid, content in artifacts.items(): packets.write_bytes(root/"artifacts"/(aid+".txt"), content) packets.write_bytes(root/"claims"/(aid+".json"), inventories[aid]) for tid, content in references.items(): packets.write_bytes(root/"public"/(tid+".txt"), content) packets.write_bytes(coordinator_dir/"source-manifest.json", design["manifest_bytes"]) for descriptor in key["inventory_copies"]: packets.write_bytes(coordinator_dir/descriptor["path"], design["inventory_copies"][descriptor["run_id"]]) packets.write_json(coordinator_dir/"claim-key.json", key) # Last: completed build record. return dict(notice=NOTICE, defaults_unfrozen=True, artifact_count=len(artifacts), blank_form_count=3) def load_key(key_path, registry_path=None): key_path = Path(key_path).resolve() key, key_bytes = read_json(key_path, "claim coordinator key") sealing.keys(key, ("claim_key_version", "notice", "coordinator_only", "defaults_unfrozen", "batch_id", "seed", "manifest_parent", "source_manifest_sha256", "registry_signature_sha256", "independent_rater_aliases", "adjudicator_alias", "artifacts", "inventory_copies", "forms"), "claim coordinator key") sealing.require(key["claim_key_version"] == "1.0", "incompatible claim key version") sealing.require(key["notice"] == NOTICE and key["coordinator_only"] is True and key["defaults_unfrozen"] is True, "retain coordinator development labels") sealing.require(type(key["seed"]) is int, "invalid presentation seed") parent = Path(sealing.text(key["manifest_parent"], "original manifest parent")) sealing.require(parent.is_absolute(), "manifest parent must be absolute") manifest_path = archived.inside(key_path.parent, "source-manifest.json") packets.hashed_bytes(manifest_path, key["source_manifest_sha256"]) copies, paths = {}, set() for row in sealing.array(key["inventory_copies"], "copied inventories"): sealing.keys(row, ("run_id", "path", "sha256"), "copied inventory") rid = sealing.text(row["run_id"], "inventory run ID") sealing.require(rid not in copies and row["path"] not in paths, "duplicate copied inventory identity/path") paths.add(row["path"]) copies[rid] = packets.hashed_bytes(archived.inside(key_path.parent, row["path"]), row["sha256"]) design = load_selection(manifest_path, registry_path, source_parent=parent, inventory_copies=copies) sealing.require(design["manifest_hash"] == key["source_manifest_sha256"], "source selection hash mismatch") sealing.require(sealing.sha(archived.registry_signature(design["registry"])) == key["registry_signature_sha256"], "incompatible registry signature") aliases = key["independent_rater_aliases"] sealing.keys(aliases, design["manifest"]["independent_rater_ids"], "independent rater mapping") issued = set() def opaque(value, prefix): sealing.require(isinstance(value, str) and re.fullmatch(prefix+r"-[0-9a-f]{32}", value) is not None and value not in issued, "invalid/duplicate opaque identity") issued.add(value) opaque(key["batch_id"], "batch") for alias in [*aliases.values(), key["adjudicator_alias"]]: opaque(alias, "rater") artifacts, claims = {}, {} source_rows = sealing.array(key["artifacts"], "key artifacts") sealing.require(len(source_rows) == len(design["selected"]), "key artifact coverage mismatch") for row, item in zip(source_rows, design["selected"]): sealing.keys(row, (*PUBLIC_ARTIFACT, "inventory_sha256", "source_run_id", "condition_id", "repetition_id", "sealed_at_utc", "inventory_state", "claims", "rating_not_before_utc"), "key artifact") source = item["source"] sealing.require(row["source_run_id"] == source["run_id"] and all(row[k] == source[k] for k in ("condition_id", "task_id", "task_version", "rubric_version", "repetition_id", "artifact_sha256")), "key artifact source identity/hash mismatch") state = "unavailable" if item["inventory"] is None else item["inventory"]["enumeration_status"] not_before = item["inventory"]["completed_at_utc"] if state == "reviewed" else item["sealed_at_utc"] sealing.require(row["inventory_state"] == state and row["family_id"] == item["family_id"] and row["sealed_at_utc"] == item["sealed_at_utc"] and row["source_packet_sha256"] == item["source_packet_sha256"] and row["rating_not_before_utc"] == not_before, "key source packet/state/time mismatch") opaque(row["artifact_id"], "artifact") expected_claims = [] if item["inventory"] is None else item["inventory"]["claims"] sealing.require(len(sealing.array(row["claims"], "key claims")) == len(expected_claims), "key claim coverage mismatch") for claim, original in zip(row["claims"], expected_claims): sealing.keys(claim, ("claim_id", "start", "end", "text", "source_claim_id"), "key claim") opaque(claim["claim_id"], "claim") sealing.require(type(claim["start"]) is int and type(claim["end"]) is int, "invalid key claim span types") sealing.require(claim["source_claim_id"] == original["claim_id"] and all(claim[k] == original[k] for k in ("start", "end", "text")), "key claim span/identity mismatch") identity = {k: row[k] for k in (*PUBLIC_ARTIFACT, "inventory_sha256")} identity.update({k: claim[k] for k in ("claim_id", "start", "end", "text")}) claims[claim["claim_id"]] = identity public_ids = iter(c["claim_id"] for c in row["claims"]) reconstructed = public_inventory(item, row["artifact_id"], lambda prefix: next(public_ids)) sealing.require(row["inventory_sha256"] == sealing.sha(sealing.json_bytes(reconstructed)), "key public inventory hash mismatch") artifacts[row["artifact_id"]] = row expected_assessors = {r: "independent" for r in aliases.values()} | {key["adjudicator_alias"]: "adjudicated"} specs = {} for spec in sealing.array(key["forms"], "key forms"): sealing.keys(spec, ("assessor_id", "form_id", "series_kind", "artifact_order"), "key form") assessor = spec["assessor_id"] sealing.require(assessor in expected_assessors and assessor not in specs and spec["series_kind"] == expected_assessors[assessor], "duplicate/unknown key form") opaque(spec["form_id"], "form") sealing.strings(spec["artifact_order"], "artifact presentation order") sealing.require(len(spec["artifact_order"]) == len(artifacts) and set(spec["artifact_order"]) == set(artifacts), "key form artifact coverage mismatch") specs[assessor] = spec sealing.require(set(specs) == set(expected_assessors), "missing key form") return key, sealing.sha(key_bytes), design, artifacts, claims, specs def load_ratings(form_paths, key, artifacts, claims, specs): series, locks, seen, provenance = {r: {} for r in specs}, {}, set(), [] now = datetime.now(timezone.utc) for path in form_paths: form, content = read_json(path, "human support form") sealing.keys(form, ("claim_form_version", "notice", "defaults_unfrozen", "batch_id", "form_id", "assessor_id", "series_kind", "locked_at_utc", "ratings"), "support form") sealing.require(form["claim_form_version"] == "1.0", "incompatible support form version") assessor = form["assessor_id"] sealing.require(assessor in specs and assessor not in seen, "unknown/duplicate assessment series") spec = specs[assessor] sealing.require(form["notice"] == NOTICE and form["defaults_unfrozen"] is True, "retain development form labels") sealing.require(form["batch_id"] == key["batch_id"] and all(form[k] == spec[k] for k in ("form_id", "series_kind")), "support form batch/series identity mismatch") lock = None if form["locked_at_utc"] is None else sealing.utc(form["locked_at_utc"], "form lock") sealing.require(lock is None or lock <= now, "future form lock") locks[assessor] = lock seen.add(assessor) provenance.append(dict(path=str(Path(path).resolve()), sha256=sealing.sha(content), assessor_id=assessor, series_kind=form["series_kind"])) for row in sealing.array(form["ratings"], "claim ratings"): sealing.keys(row, RATING_FIELDS, "claim rating") cid = row["claim_id"] sealing.require(cid in claims and cid not in series[assessor], "unknown/duplicate assessed claim") sealing.require(type(row["start"]) is int and type(row["end"]) is int, "invalid rating claim span types") sealing.require(all(row[k] == claims[cid][k] for k in CLAIM_IDENTITY), "claim/artifact/source/inventory identity or hash mismatch") judgment = row["judgment"] sealing.require(judgment is None or judgment in JUDGMENTS, "invalid human support judgment") sealing.strings(row["evidence_locations"], "human evidence locations") if judgment is None: sealing.require(row["completed_at_utc"] is None and row["independent_judgments"] is None and row["resolution_note"] is None, "missing judgment marked completed/resolved") sealing.require(row["reason"] is None or isinstance(row["reason"], str), "invalid pending reason") else: sealing.text(row["reason"], "human support rationale") sealing.require(row["evidence_locations"], "support judgment needs evidence locations") stamp = sealing.utc(row["completed_at_utc"], "support judgment time") artifact = artifacts[row["artifact_id"]] sealing.require(lock is not None and sealing.utc(artifact["rating_not_before_utc"], "inventory/seal time") <= stamp <= lock, "judgment must follow sealing/inventory review and precede form lock") if form["series_kind"] == "independent": sealing.require(row["independent_judgments"] is None and row["resolution_note"] is None, "independent form contains adjudication fields") series[assessor][cid] = row raters = list(key["independent_rater_aliases"].values()) for cid, row in series[key["adjudicator_alias"]].items(): if row["judgment"] is None: continue sealing.keys(row["independent_judgments"], raters, "adjudication basis") sealing.text(row["resolution_note"], "human adjudication resolution") for rater in raters: original = series[rater].get(cid) sealing.require(original is not None and original["judgment"] is not None, "adjudication requires both independent judgments") sealing.require(row["independent_judgments"][rater] == original["judgment"], "stale adjudication basis") sealing.require(sealing.utc(row["completed_at_utc"], "adjudication time") >= locks[rater], "adjudication predates independent lock") return series, provenance def summarize_group(artifacts, ratings): rows = [] for artifact in artifacts: state = artifact["inventory_state"] expected = [claim["claim_id"] for claim in artifact["claims"]] judged = [ratings[cid] for cid in expected if cid in ratings and ratings[cid]["judgment"] is not None] unsupported = sum(r["judgment"] == "unsupported" for r in judged) complete = state == "reviewed" and len(judged) == len(expected) rows.append(dict(artifact_id=artifact["artifact_id"], inventory_state=state, enumerated_claim_count=None if state == "unavailable" else len(expected), assessed_claims=len(judged), missing_judgments=None if state == "unavailable" else len(expected)-len(judged), unsupported_claims=unsupported, declared_coverage_complete=complete, unsupported_claim_rate=unsupported/len(judged) if complete and judged else None)) known = sum(r["enumerated_claim_count"] for r in rows if r["enumerated_claim_count"] is not None) assessed = sum(r["assessed_claims"] for r in rows) unsupported = sum(r["unsupported_claims"] for r in rows) complete = all(r["declared_coverage_complete"] for r in rows) return dict(declared_artifacts=len(rows), reviewed_inventories=sum(r["inventory_state"] == "reviewed" for r in rows), reviewed_no_claim_artifacts=sum(r["inventory_state"] == "reviewed" and r["enumerated_claim_count"] == 0 for r in rows), unfinished_inventories=sum(r["inventory_state"] == "unfinished" for r in rows), unavailable_inventories=sum(r["inventory_state"] == "unavailable" for r in rows), provided_claims=known, assessed_claims=assessed, unsupported_claims=unsupported, missing_judgments=known-assessed, judgment_coverage=dict(numerator=assessed, denominator=known, rate=None if known == 0 else assessed/known), declared_coverage_complete=complete, complete_rate_withheld_for_missing_coverage=not complete, unsupported_claim_rate=unsupported/assessed if complete and assessed else None, artifacts=rows) def agreement_group(artifacts, a, b): ids = [c["claim_id"] for artifact in artifacts for c in artifact["claims"]] common = [cid for cid in ids if a.get(cid, {}).get("judgment") is not None and b.get(cid, {}).get("judgment") is not None] agree = sum(a[cid]["judgment"] == b[cid]["judgment"] for cid in common) return dict(agreeing_claims=agree, jointly_assessed_claim_denominator=len(common), provided_claims_missing_joint_judgments=len(ids)-len(common), raw_agreement=agree/len(common) if common else None, unfinished_inventories=sum(r["inventory_state"] == "unfinished" for r in artifacts), unavailable_inventories=sum(r["inventory_state"] == "unavailable" for r in artifacts)) def summarize(key_path, form_paths, registry_path=None): key, key_hash, design, artifacts, claims, specs = load_key(key_path, registry_path) series, sources = load_ratings(form_paths, key, artifacts, claims, specs) values = list(artifacts.values()) conditions = sorted({a["condition_id"] for a in values}) def grouped(function): return dict(overall=function(values), by_condition={c: function([a for a in values if a["condition_id"] == c]) for c in conditions}) raters = list(key["independent_rater_aliases"].values()) return dict(claim_support_report_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, coordinator_key_sha256=key_hash, source_manifest_sha256=design["manifest_hash"], registry_signature_sha256=key["registry_signature_sha256"], archives=design["archives"], unselected_archived_run_ids=design["unselected_archived_run_ids"], scope="explicitly selected final artifacts and human-enumerated claim occurrences; not exhaustive factual-error detection", rate_policy="unsupported judgments / all nonnull support judgments; complete rates withheld for unfinished/unavailable inventories or missing judgments; claim-weighted, not the primary quality score", independent={r: grouped(lambda rows: summarize_group(rows, series[r])) for r in raters}, adjudicated=grouped(lambda rows: summarize_group(rows, series[key["adjudicator_alias"]])), independent_rater_agreement=grouped(lambda rows: agreement_group(rows, series[raters[0]], series[raters[1]])), assessment_sources=sources) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="command", required=True) prepare = sub.add_parser("build") prepare.add_argument("--manifest", type=Path, required=True) prepare.add_argument("--rater-output", type=Path, required=True) prepare.add_argument("--coordinator-output", type=Path, required=True) prepare.add_argument("--seed", type=int, default=0) report = sub.add_parser("summarize") report.add_argument("--key", type=Path, required=True) report.add_argument("--forms", type=Path, nargs="*", default=[]) for child in (prepare, report): child.add_argument("--registry", type=Path) args = parser.parse_args(argv) try: result = (build(args.manifest, args.rater_output, args.coordinator_output, args.registry, args.seed) if args.command == "build" else summarize(args.key, args.forms, args.registry)) except (sealing.ValidationError, sealing.account_resources.LedgerError, OSError, KeyError, TypeError, ValueError, ArithmeticError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())