#!/usr/bin/env python3 """Offline assessed requirement regression on explicitly paired sealed snapshots. Development tooling: no history selection, semantic regrading, or model calls. """ import argparse from datetime import datetime, timezone import hashlib import json from pathlib import Path import sys import tempfile import score_forms as scoring BASE = Path(__file__).resolve().parent NOTICE = "Development requirement-regression tooling, not empirical evidence; transitions reflect supplied assessments only." def read_snapshot(descriptor, parent): scoring.keys(descriptor, ("path", "sha256"), "assessment source descriptor") path = (parent/scoring.text(descriptor["path"], "assessment path")).resolve() scoring.digest(descriptor["sha256"], "assessment source sha256") content = path.read_bytes() scoring.require(hashlib.sha256(content).hexdigest() == descriptor["sha256"], "assessment source hash mismatch") return path, content def load_set(spec, parent, registry_path, tasks, mapping, analysis_raters, run_seal_time): """Validate exact source snapshots through the unchanged primary validator.""" scoring.keys(spec, ("coordinator_manifest", "rating_forms", "adjudication_forms", "rater_mapping"), "assessment set") aliases = spec["rater_mapping"] scoring.keys(aliases, analysis_raters, "analysis-to-source rater mapping") for value in aliases.values(): scoring.text(value, "source rater alias") scoring.require(len(set(aliases.values())) == 2, "rater mapping must not pool identities") descriptors = [("manifest.json", spec["coordinator_manifest"])] descriptors += [(f"rating-{i}.json", d) for i, d in enumerate(scoring.array(spec["rating_forms"], "rating forms"))] descriptors += [(f"adjudication-{i}.json", d) for i, d in enumerate(scoring.array(spec["adjudication_forms"], "adjudication forms"))] provenance = [] with tempfile.TemporaryDirectory(prefix="regression-validation-snapshot-") as tmp: frozen = Path(tmp) for filename, descriptor in descriptors: path, content = read_snapshot(descriptor, parent) (frozen/filename).write_bytes(content) provenance.append(dict(path=str(path), sha256=descriptor["sha256"])) coordinator = scoring.read_json(frozen/"manifest.json") source_raters, _, assignments, _ = scoring.manifest_data(coordinator, tasks) scoring.require(set(aliases.values()) == set(source_raters), "rater mapping differs from source coordinator manifest") rating_paths = [frozen/name for name, _ in descriptors if name.startswith("rating-")] adjudication_paths = [frozen/name for name, _ in descriptors if name.startswith("adjudication-")] # Its complete form/identity/mapping/link validator is reused. Aggregate # primary scores are neither changed nor used to infer criterion changes. scoring.score_assessment(registry_path, frozen/"manifest.json", rating_paths, adjudication_paths) independent = {r: {} for r in source_raters} adjudicated = {} for paths, member, status, rid, judgment in ( (rating_paths, "ratings", "assessment_status", "rating_record_id", "satisfaction"), (adjudication_paths, "adjudications", "adjudication_status", "adjudication_record_id", "adjudicated_satisfaction")): for path in paths: form = scoring.read_json(path) lookup = independent[form["rater_id"]] if member == "ratings" else adjudicated for row in form[member]: scoring.require(scoring.utc(row["completed_at_utc"], "assessment completion") >= run_seal_time, "assessment predates declared run-batch seal") issue = row["assessment_issue"] if member == "ratings" else row["adjudication_issue"] lookup[row["blinded_artifact_id"]] = dict( record_id=row[rid], status=row[status], issue=issue, values=[scoring.binary(c[judgment], "criterion judgment") for c in scoring.criterion_rows(row, mapping)]) return dict(assignments=assignments, independent=independent, adjudicated=adjudicated, rater_mapping=aliases, provenance=provenance, source_batch_id=coordinator["batch_id"], source_seal_record_id=coordinator["seal_record_id"]) def validate_pairs(pairs, tasks, earlier, final): scoring.require(scoring.array(pairs, "pairs"), "declare at least one pair") pair_ids, run_ids, cells = set(), set(), set() endpoints = {"earlier": set(), "final": set()} for pair in pairs: scoring.keys(pair, ("pair_id", "run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id", "earlier", "final"), "paired run") for key in ("pair_id", "run_id", "condition_id", "repetition_id"): scoring.text(pair[key], key) cell = tuple(pair[k] for k in ("condition_id", "task_id", "repetition_id")) scoring.require(pair["pair_id"] not in pair_ids and pair["run_id"] not in run_ids and cell not in cells, "duplicate pair, run, or condition/task/repetition cell") pair_ids.add(pair["pair_id"]) run_ids.add(pair["run_id"]) cells.add(cell) scoring.require(pair["task_id"] in tasks, "unknown pair task") task = tasks[pair["task_id"]] scoring.require(all(pair[k] == task[k] for k in ("task_version", "rubric_version")), "incompatible pair task/rubric version") for phase, source in (("earlier", earlier), ("final", final)): endpoint = pair[phase] scoring.keys(endpoint, ("blinded_artifact_id", "blinded_artifact_sha256", "turn"), phase+" endpoint") aid = scoring.text(endpoint["blinded_artifact_id"], "endpoint artifact ID") scoring.digest(endpoint["blinded_artifact_sha256"], "endpoint sha256") scoring.require(type(endpoint["turn"]) is int and endpoint["turn"] >= 1, "endpoint turns must be positive integers") scoring.require(aid not in endpoints[phase], "duplicate endpoint assignment across pairs") endpoints[phase].add(aid) scoring.require(aid in source["assignments"], "endpoint identity missing from source coordinator manifest") assignment = source["assignments"][aid] scoring.require(all(assignment[k] == pair[k] for k in ("condition_id", "task_id", "task_version", "rubric_version", "repetition_id")) and assignment["family_id"] == task["family_id"] and assignment["blinded_artifact_sha256"] == endpoint["blinded_artifact_sha256"], "endpoint task/rubric/run-cell/hash mismatch") scoring.require(pair["earlier"]["turn"] < pair["final"]["turn"], "reversed or equal chronological endpoints") scoring.require(pair["earlier"]["blinded_artifact_id"] != pair["final"]["blinded_artifact_id"], "earlier and final require distinct snapshot identities") def pair_result(pair, mapping, earlier_record, final_record): criteria = [] for index, (criterion, requirement) in enumerate(mapping): old = None if earlier_record is None else earlier_record["values"][index] new = None if final_record is None else final_record["values"][index] observed = old is not None and new is not None criteria.append(dict(criterion_id=criterion, public_requirement_id=requirement, earlier_satisfaction=old, final_satisfaction=new, paired_judgments_observed=observed, transition=f"{old}->{new}" if observed else None, regressed=(old == 1 and new == 0) if observed else None, eligible_previously_satisfied=(observed and old == 1))) return dict(pair_id=pair["pair_id"], run_id=pair["run_id"], condition_id=pair["condition_id"], task_id=pair["task_id"], task_version=pair["task_version"], rubric_version=pair["rubric_version"], repetition_id=pair["repetition_id"], earlier=pair["earlier"], final=pair["final"], earlier_record=None if earlier_record is None else {k: earlier_record[k] for k in ("record_id", "status", "issue")}, final_record=None if final_record is None else {k: final_record[k] for k in ("record_id", "status", "issue")}, criteria=criteria) def summarize(results): rows = [c for pair in results for c in pair["criteria"]] observed = sum(c["paired_judgments_observed"] for c in rows) denominator = sum(c["eligible_previously_satisfied"] for c in rows) regressions = sum(c["regressed"] is True for c in rows) return dict(declared_artifact_pairs=len(results), expected_criterion_pairs=len(rows), observed_criterion_pairs=observed, missing_criterion_pairs=len(rows)-observed, missing_endpoint_record_pairs=sum(p["earlier_record"] is None or p["final_record"] is None for p in results), complete_criterion_pair_coverage=observed == len(rows), known_earlier_satisfied=sum(c["earlier_satisfaction"] == 1 for c in rows), missing_final_for_known_satisfied=sum(c["earlier_satisfaction"] == 1 and c["final_satisfaction"] is None for c in rows), eligible_previously_satisfied_denominator=denominator, observed_regressions=regressions, assessed_regression_rate=None if denominator == 0 else regressions/denominator, rate_scope="observed binary criterion pairs only; no missing-judgment imputation") def analyze_regression(manifest_path, registry_path=None): manifest_path = Path(manifest_path).resolve() registry_path = Path(registry_path or BASE/"task-registry.json").resolve() manifest = scoring.read_json(manifest_path) scoring.keys(manifest, ("manifest_version", "analysis_id", "run_batch_seal", "independent_rater_ids", "earlier_assessments", "final_assessments", "pairs"), "regression manifest") scoring.require(manifest["manifest_version"] == "1.0", "unsupported regression manifest version") scoring.text(manifest["analysis_id"], "analysis_id") seal = manifest["run_batch_seal"] scoring.keys(seal, ("seal_record_id", "sealed", "sealed_at_utc"), "run batch seal") scoring.text(seal["seal_record_id"], "run seal ID") scoring.require(seal["sealed"] is True, "run batch must be explicitly declared sealed") seal_time = scoring.utc(seal["sealed_at_utc"], "run batch sealed_at_utc") scoring.require(seal_time <= datetime.now(timezone.utc), "future run-batch seal") raters = manifest["independent_rater_ids"] scoring.strings(raters, "analysis rater IDs") scoring.require(len(raters) == len(set(raters)) == 2, "declare two distinct analysis rater identities") registry = scoring.read_json(registry_path) scoring.require(registry["kit_version"] == "1.0", "unsupported registry version") tasks, _, mapping = scoring.registry_data(registry, registry_path.parent) earlier = load_set(manifest["earlier_assessments"], manifest_path.parent, registry_path, tasks, mapping, raters, seal_time) final = load_set(manifest["final_assessments"], manifest_path.parent, registry_path, tasks, mapping, raters, seal_time) validate_pairs(manifest["pairs"], tasks, earlier, final) def series(old_lookup, new_lookup): results = [pair_result(p, mapping, old_lookup.get(p["earlier"]["blinded_artifact_id"]), new_lookup.get(p["final"]["blinded_artifact_id"])) for p in manifest["pairs"]] for result in results: result["summary"] = summarize([result]) conditions = sorted({p["condition_id"] for p in results}) return dict(pairs=results, totals=summarize(results), by_condition={c: summarize([p for p in results if p["condition_id"] == c]) for c in conditions}) return dict(regression_report_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, analysis_id=manifest["analysis_id"], run_batch_seal=seal, comparison_policy="one explicitly designated earlier snapshot versus the designated final snapshot per run", manifest_sha256=hashlib.sha256(manifest_path.read_bytes()).hexdigest(), registry_sha256=hashlib.sha256(registry_path.read_bytes()).hexdigest(), source_provenance={phase: {k: source[k] for k in ("source_batch_id", "source_seal_record_id", "rater_mapping", "provenance")} for phase, source in (("earlier", earlier), ("final", final))}, independent={r: series(earlier["independent"][earlier["rater_mapping"][r]], final["independent"][final["rater_mapping"][r]]) for r in raters}, adjudicated=series(earlier["adjudicated"], final["adjudicated"])) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--registry", type=Path, default=BASE/"task-registry.json") args = parser.parse_args(argv) try: report = analyze_regression(args.manifest, args.registry) except (scoring.ValidationError, OSError, KeyError, TypeError, ValueError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(report, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())