#!/usr/bin/env python3 """Read-only, standard-library arithmetic for manual development assessments. No artifact generation, model calls, semantic grading, or experimental execution. """ import argparse from datetime import datetime, timezone from fractions import Fraction import hashlib import json import math from pathlib import Path import re import sys BASE = Path(__file__).resolve().parent NOTICE = "Development scoring tooling; arithmetic output alone is not empirical evidence." class ValidationError(ValueError): pass def require(test, message): if not test: raise ValidationError(message) def text(value, label): require(isinstance(value, str) and bool(value.strip()), f"{label}: expected nonempty string") return value def keys(obj, expected, label): require(isinstance(obj, dict) and set(obj) == set(expected), f"{label}: unexpected/missing fields") def binary(value, label): require(value is None or (type(value) is int and value in (0, 1)), f"{label}: expected integer 0, integer 1, or null") return value def strings(value, label): require(isinstance(value, list), f"{label}: expected array") for item in value: text(item, label) def array(value, label): require(isinstance(value, list), f"{label}: expected array") return value def utc(value, label): text(value, label) try: result = datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError as exc: raise ValidationError(f"{label}: invalid UTC timestamp") from exc require(result.tzinfo is not None and result.utcoffset() == timezone.utc.utcoffset(result), f"{label}: expected UTC timestamp") return result def digest(value, label): require(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None, f"{label}: expected lowercase SHA-256") def read_json(path): def unique(pairs): result = {} for key, value in pairs: require(key not in result, f"{path}: duplicate JSON key {key}") result[key] = value return result def nonfinite(value): raise ValidationError(f"{path}: non-finite JSON number {value}") try: return json.loads(Path(path).read_text(encoding="utf-8"), object_pairs_hook=unique, parse_constant=nonfinite) except (OSError, json.JSONDecodeError) as exc: raise ValidationError(f"{path}: {exc}") from exc def registry_data(registry, base): tasks = {} for task in registry["tasks"]: tid = text(task["task_id"], "registry task_id") require(tid not in tasks, "duplicate registry task") tasks[tid] = task for field in ("public_packet", "assessor_guide"): asset = task[field] require(hashlib.sha256((base / asset["path"]).read_bytes()).hexdigest() == asset["sha256"], f"registry hash mismatch: {tid} {field}") conventions = registry["common_conventions"] require(hashlib.sha256((base / conventions["path"]).read_bytes()).hexdigest() == conventions["sha256"], "common conventions hash mismatch") mapping = [(f"C{i}", f"R{i}") for i in range(1, 13)] require(registry["criterion_mapping"] == [dict(criterion_id=c, public_requirement_id=r) for c, r in mapping], "registry criterion mapping must be C1–C12 to R1–R12") families = {f["family_id"]: f["task_ids"] for f in registry["families"]} require(len(families) == len(registry["families"]) == 3 and len(tasks) == 12, "expected three families and twelve tasks") members = [tid for group in families.values() for tid in group] require(len(members) == len(set(members)) == 12 and set(members) == set(tasks), "invalid registry family membership") for family, group in families.items(): require(len(group) == 4 and all(tasks[t]["family_id"] == family for t in group), "expected four matching tasks per family") return tasks, families, mapping IDENTITY = ("blinded_artifact_id", "blinded_artifact_sha256", "task_id", "task_version", "rubric_version", "family_id") def identity(record, expected): require(all(record[k] == expected[k] for k in IDENTITY), f"artifact identity/version mismatch: {record.get('blinded_artifact_id')}") def manifest_data(manifest, tasks): keys(manifest, ("manifest_version", "batch_id", "seal_record_id", "independent_rater_ids", "conditions", "assignments"), "manifest") require(manifest["manifest_version"] == "1.0", "unsupported manifest version") text(manifest["batch_id"], "batch_id") text(manifest["seal_record_id"], "seal_record_id") raters = manifest["independent_rater_ids"] strings(raters, "independent_rater_ids") require(len(raters) == len(set(raters)) == 2, "declare exactly two distinct independent raters") conditions = {} for condition in array(manifest["conditions"], "conditions"): keys(condition, ("condition_id", "repetition_ids"), "condition") cid = text(condition["condition_id"], "condition_id") repetitions = condition["repetition_ids"] strings(repetitions, "repetition_ids") require(repetitions and len(repetitions) == len(set(repetitions)), "empty/duplicate repetitions") require(cid not in conditions, "duplicate condition") conditions[cid] = repetitions require(conditions, "declare at least one condition") assignments, cells = {}, {} for row in array(manifest["assignments"], "assignments"): keys(row, (*IDENTITY, "condition_id", "repetition_id"), "assignment") aid = text(row["blinded_artifact_id"], "blinded_artifact_id") digest(row["blinded_artifact_sha256"], aid) require(row["task_id"] in tasks, "unknown manifest task") task = tasks[row["task_id"]] require(all(row[k] == task[k] for k in ("task_version", "rubric_version", "family_id")), "manifest task/rubric/family mismatch") cid, rep = row["condition_id"], row["repetition_id"] require(cid in conditions and rep in conditions[cid], "undeclared condition/repetition") cell = (cid, row["task_id"], rep) require(aid not in assignments and cell not in cells, "duplicate manifest artifact or task/repetition cell") assignments[aid], cells[cell] = row, aid return raters, conditions, assignments, cells def scores(values, supplied): keys(supplied, ("criterion_count", "satisfied_criteria", "requirement_satisfaction_score"), "scores") require(type(supplied["criterion_count"]) is int and supplied["criterion_count"] == 12, "criterion_count must be 12") count = None if None in values else sum(values) score = None if count is None else Fraction(count, 12) reported_count, reported_score = supplied["satisfied_criteria"], supplied["requirement_satisfaction_score"] if reported_count is not None: require(type(reported_count) is int and reported_count == count, "stored satisfied count mismatch") if reported_score is not None: require(type(reported_score) in (int, float) and math.isfinite(reported_score) and score is not None and math.isclose(reported_score, float(score), rel_tol=0, abs_tol=1e-12), "stored score mismatch") return score def evidence(row, reason, judged): for field in ("artifact_evidence_locations", "public_basis_locations"): strings(row[field], field) if judged: require(row[field], f"judged criterion needs {field}") require(row[reason] is None or isinstance(row[reason], str), f"invalid {reason}") if judged: text(row[reason], reason) def criterion_rows(record, mapping): rows = record["criteria"] require(isinstance(rows, list) and len(rows) == 12, "expected exactly twelve criteria") by_id = {} for row in rows: cid = row["criterion_id"] require(cid not in by_id, "duplicate criterion") by_id[cid] = row require(set(by_id) == {c for c, _ in mapping}, "unknown/missing criterion") require(all(by_id[c]["public_requirement_id"] == r for c, r in mapping), "criterion/requirement mismatch") return [by_id[c] for c, _ in mapping] def score_assessment(registry_path, manifest_path, rating_paths, adjudication_paths=()): registry_path = Path(registry_path) tasks, families, mapping = registry_data(read_json(registry_path), registry_path.parent) manifest = read_json(manifest_path) raters, conditions, assignments, cells = manifest_data(manifest, tasks) forms, records, ratings, adjudications = {}, {}, {}, {} def form_header(form, kind): common = {"form_version", "form_type", "asset_status", "task_registry", "form_id", "batch_id", "seal_record_id", "locked_at_utc"} extra = {"rater_id", "ratings"} if kind == "independent_rating_batch" else {"adjudicator_id", "source_rating_form_ids", "independent_rater_ids", "adjudications"} keys(form, common | extra, "form (method metadata belongs only in manifest)") require(form["form_version"] == "1.0" and form["form_type"] == kind, "unsupported form type/version") fid = text(form["form_id"], "form_id") require(fid not in forms, "duplicate form ID") require(all(form[k] == manifest[k] for k in ("batch_id", "seal_record_id")), "form batch/seal mismatch") lock = utc(form["locked_at_utc"], "locked_at_utc") forms[fid] = form return fid, lock def record_header(row, rid_key, status_key, issue_key, lock): expected = {*IDENTITY, rid_key, status_key, issue_key, "completed_at_utc", "criteria", "scores"} if rid_key == "adjudication_record_id": expected.add("source_rating_record_ids") keys(row, expected, "record") rid = text(row[rid_key], rid_key) require(rid not in records, "duplicate record ID") records[rid] = row aid = row["blinded_artifact_id"] require(aid in assignments, "record artifact absent from manifest") identity(row, assignments[aid]) require(row[status_key] in ("complete", "needs_resolution"), "record status must be complete or needs_resolution") require(row[issue_key] is None or isinstance(row[issue_key], str), "invalid issue description") completion = utc(row["completed_at_utc"], "completed_at_utc") require(completion <= lock, "record completed after form lock") return rid, aid, completion for path in rating_paths: form = read_json(path) fid, lock = form_header(form, "independent_rating_batch") rater = form["rater_id"] require(rater in raters, "undeclared independent rater") for row in array(form["ratings"], "ratings"): rid, aid, _ = record_header(row, "rating_record_id", "assessment_status", "assessment_issue", lock) require((rater, aid) not in ratings, "duplicate rater/artifact record") values = [] for c in criterion_rows(row, mapping): keys(c, ("criterion_id", "public_requirement_id", "satisfaction", "artifact_evidence_locations", "public_basis_locations", "reason", "ambiguity_note"), "rating criterion") value = binary(c["satisfaction"], rid) evidence(c, "reason", value is not None) require(c["ambiguity_note"] is None or isinstance(c["ambiguity_note"], str), "invalid ambiguity note") values.append(value) score = scores(values, row["scores"]) require(row["assessment_status"] != "complete" or score is not None, "complete rating has missing values") ratings[(rater, aid)] = dict(record_id=rid, form_id=fid, lock=lock, values=values, score=score, eligible=score is not None and row["assessment_status"] == "complete") for path in adjudication_paths: form = read_json(path) _, lock = form_header(form, "adjudication_batch") require(text(form["adjudicator_id"], "adjudicator_id") not in raters, "third adjudicator must differ from both raters") source_raters, source_forms = form["independent_rater_ids"], form["source_rating_form_ids"] strings(source_raters, "independent_rater_ids") strings(source_forms, "source_rating_form_ids") require(len(source_raters) == 2 and set(source_raters) == set(raters) and len(source_forms) == 2, "invalid source rater/form pair") for rater, fid in zip(source_raters, source_forms): require(fid in forms and forms[fid].get("rater_id") == rater, "source rating form/rater mismatch") for row in array(form["adjudications"], "adjudications"): rid, aid, completion = record_header(row, "adjudication_record_id", "adjudication_status", "adjudication_issue", lock) require(aid not in adjudications, "duplicate adjudicated artifact") sources = [ratings.get((r, aid)) for r in source_raters] require(all(s is not None for s in sources), "adjudication requires both source records, including missing judgments") require(row["source_rating_record_ids"] == [s["record_id"] for s in sources] and source_forms == [s["form_id"] for s in sources], "source record/form mismatch") require(all(completion >= s["lock"] for s in sources), "adjudication predates independent locks") values, resolutions = [], [] for index, c in enumerate(criterion_rows(row, mapping)): keys(c, ("criterion_id", "public_requirement_id", "independent_satisfaction", "disagreement", "resolution", "adjudicated_satisfaction", "artifact_evidence_locations", "public_basis_locations", "resolution_rationale"), "adjudication criterion") copied = c["independent_satisfaction"] require(isinstance(copied, list) and len(copied) == 2, "invalid copied judgment pair") for value in copied: binary(value, rid) require(copied == [s["values"][index] for s in sources], "copied independent judgments mismatch") disagreement = None if None in copied else copied[0] != copied[1] require(type(c["disagreement"]) is type(disagreement) and c["disagreement"] == disagreement, "incorrect disagreement flag") value = binary(c["adjudicated_satisfaction"], rid) route = c["resolution"] if value is None: require(route in (None, "unresolved"), "missing final judgment needs unresolved route") elif disagreement is False: require(route == "agreement_carried_forward" and value == copied[0], "agreed judgment must be carried forward") elif disagreement is True: require(route == "adjudicator_decision", "disagreement needs adjudicator decision") else: require(route == "missing_or_ambiguous_rating_resolved", "missing rating needs explicit resolution route") evidence(c, "resolution_rationale", value is not None) values.append(value) resolutions.append(route) score = scores(values, row["scores"]) require(row["adjudication_status"] != "complete" or score is not None, "complete adjudication has missing values") adjudications[aid] = dict(record_id=rid, values=values, score=score, resolutions=resolutions, eligible=score is not None and row["adjudication_status"] == "complete") def number(score): return None if score is None else float(score) def series(lookup): output = {} for cid, repetitions in conditions.items(): task_scores, missing_assignments, missing_records, incomplete = {}, [], [], [] observed = complete = 0 for tid in tasks: repeats = [] for rep in repetitions: aid = cells.get((cid, tid, rep)) if aid is None: missing_assignments.append(dict(task_id=tid, repetition_id=rep)) record = lookup.get(aid) if aid is not None and record is None: missing_records.append(aid) if record is not None: observed += 1 if record["eligible"]: complete += 1 else: incomplete.append(dict(blinded_artifact_id=aid, missing_criterion_ids=[f"C{i+1}" for i, v in enumerate(record["values"]) if v is None])) repeats.append(record["score"] if record is not None and record["eligible"] else None) task_scores[tid] = None if None in repeats else sum(repeats, Fraction()) / len(repeats) expected = len(tasks) * len(repetitions) covered = complete == expected family_scores = {f: (sum((task_scores[t] for t in group), Fraction()) / len(group) if covered else None) for f, group in families.items()} overall = sum(family_scores.values(), Fraction()) / len(families) if covered else None output[cid] = dict(coverage_complete=covered, expected_artifacts=expected, observed_records=observed, complete_records=complete, missing_assignments=missing_assignments, missing_record_artifact_ids=missing_records, incomplete_records=incomplete, task_means={t: number(s) for t, s in task_scores.items()}, family_scores={f: number(s) for f, s in family_scores.items()}, overall_score=number(overall)) return output def agreement(condition_ids): expected = matches = denominator = 0 for cid in condition_ids: for tid in tasks: for rep in conditions[cid]: expected += 12 aid = cells.get((cid, tid, rep)) pair = [ratings.get((r, aid)) for r in raters] if all(p is not None for p in pair): for left, right in zip(pair[0]["values"], pair[1]["values"]): if left is not None and right is not None: denominator += 1 matches += left == right return dict(matching_pairs=matches, compared_pairs=denominator, disagreement_pairs=denominator-matches, expected_pairs=expected, missing_pairs=expected-denominator, raw_agreement=None if denominator == 0 else matches/denominator) def artifact_scores(lookup): return [dict(blinded_artifact_id=aid, record_id=record["record_id"], condition_id=assignments[aid]["condition_id"], task_id=assignments[aid]["task_id"], repetition_id=assignments[aid]["repetition_id"], score=number(record["score"]), eligible_for_aggregation=record["eligible"], missing_criterion_ids=[f"C{i+1}" for i, v in enumerate(record["values"]) if v is None]) for aid, record in sorted(lookup.items())] independent = {} for rater in raters: lookup = {aid: row for (r, aid), row in ratings.items() if r == rater} independent[rater] = dict(artifact_scores=artifact_scores(lookup), conditions=series(lookup)) return dict(report_version="1.0", notice=NOTICE, coordinator_only=True, batch_id=manifest["batch_id"], seal_record_id=manifest["seal_record_id"], independent=independent, adjudicated=dict(artifact_scores=artifact_scores(adjudications), conditions=series(adjudications)), raw_independent_agreement=dict(overall=agreement(conditions), conditions={c: agreement([c]) for c in conditions})) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--registry", type=Path, default=BASE / "task-registry.json") parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--ratings", type=Path, nargs="+", required=True) parser.add_argument("--adjudications", type=Path, nargs="*", default=[]) args = parser.parse_args(argv) try: report = score_assessment(args.registry, args.manifest, args.ratings, args.adjudications) except (ValidationError, OSError, KeyError, TypeError, ValueError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(report, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())