#!/usr/bin/env python3 """Offline retrospective requirement-satisfaction curves from sealed snapshots. Development tooling; no judgments, carry-forward, best-snapshot or endpoint selection. """ import argparse from datetime import datetime, timezone import json import math import os from pathlib import Path import sys import tempfile BASE = Path(__file__).resolve().parent sys.path.insert(0, str(BASE.parent/"repeated-work")) import assess_repeated_work as chronology shared, archived, sealing, packets, scoring = chronology.shared, chronology.archived, chronology.sealing, chronology.packets, chronology.scoring import analyze_requirement_regression as assessment_helpers NOTICE = "Development retrospective progress tooling, not empirical evidence; defaults unfrozen; existing human judgments only; never adaptation feedback." RUN_ID = ("archive_id", "seal_record_id", "run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id") def load_assessment_set(spec, parent, registry_path, tasks, raters): sealing.keys(spec, ("assessment_set_id", "coordinator_manifest", "rating_forms", "adjudication_forms", "rater_mapping"), "assessment set") sealing.text(spec["assessment_set_id"], "assessment set ID") aliases = spec["rater_mapping"] sealing.keys(aliases, raters, "analysis-to-source rater mapping") for alias in aliases.values(): sealing.text(alias, "source rater alias") sealing.require(len(set(aliases.values())) == 2, "rater mapping must not pool identities") descriptors = [("coordinator.json", spec["coordinator_manifest"])] descriptors += [(f"rating-{i}.json", d) for i, d in enumerate(sealing.array(spec["rating_forms"], "rating forms"))] descriptors += [(f"adjudication-{i}.json", d) for i, d in enumerate(sealing.array(spec["adjudication_forms"], "adjudication forms"))] provenance, metadata = [], {} with tempfile.TemporaryDirectory(prefix="progress-assessment-validation-") as temporary: root = Path(temporary) for filename, descriptor in descriptors: path, content = assessment_helpers.read_snapshot(descriptor, parent) (root/filename).write_bytes(content) # Only private temporary copies. provenance.append(dict(path=str(path), sha256=descriptor["sha256"])) coordinator = scoring.read_json(root/"coordinator.json") source_raters, _, assignments, _ = scoring.manifest_data(coordinator, tasks) sealing.require(set(aliases.values()) == set(source_raters), "source rater mapping mismatch") rating_paths = [root/n for n, _ in descriptors if n.startswith("rating-")] adjudication_paths = [root/n for n, _ in descriptors if n.startswith("adjudication-")] # Reuse the unchanged complete primary validator and normalized scores. report = scoring.score_assessment(registry_path, root/"coordinator.json", rating_paths, adjudication_paths) now = datetime.now(timezone.utc) for paths, member, actor, status in ((rating_paths, "ratings", "rater_id", "assessment_status"), (adjudication_paths, "adjudications", "adjudicator_id", "adjudication_status")): for path in paths: form = scoring.read_json(path) sealing.require(scoring.utc(form["locked_at_utc"], "assessment form lock") <= now, "future assessment lock") for row in form[member]: stamp = scoring.utc(row["completed_at_utc"], "assessment completion") sealing.require(stamp <= now, "future assessment completion") owner = form[actor] if member == "ratings" else "adjudicated" metadata[(owner, row["blinded_artifact_id"])] = dict(status=row[status], completed_at_utc=row["completed_at_utc"]) def records(rows, owner): result = {} for row in rows: aid = row["blinded_artifact_id"] result[aid] = dict(record_id=row["record_id"], score=row["score"] if row["eligible_for_aggregation"] else None, eligible=row["eligible_for_aggregation"], missing_criterion_ids=row["missing_criterion_ids"], observed_criterion_judgments=12-len(row["missing_criterion_ids"]), **metadata[(owner, aid)]) return result return dict(assignments=assignments, independent={r: records(report["independent"][aliases[r]]["artifact_scores"], aliases[r]) for r in raters}, adjudicated=records(report["adjudicated"]["artifact_scores"], "adjudicated"), rater_mapping=aliases, source_batch_id=coordinator["batch_id"], source_seal_record_id=coordinator["seal_record_id"], provenance=provenance) def load_analysis(manifest_path, registry_path=None): manifest_path = Path(manifest_path).resolve() parent = manifest_path.parent manifest, content = shared.read_json(manifest_path, "progress manifest") sealing.keys(manifest, ("progress_manifest_version", "analysis_id", "turn_count", "independent_rater_ids", "conditions", "archives", "assessment_sets", "runs"), "progress manifest") sealing.require(manifest["progress_manifest_version"] == "1.0", "incompatible progress manifest version") sealing.text(manifest["analysis_id"], "analysis ID") turn_count = chronology.integer(manifest["turn_count"], 1, "declared turn count") raters = manifest["independent_rater_ids"] sealing.strings(raters, "independent analysis raters") sealing.require(len(raters) == len(set(raters)) == 2, "declare two distinct independent analysis raters") registry_path = Path(registry_path or sealing.KIT/"task-registry.json").resolve() registry = scoring.read_json(registry_path) sealing.require(registry["kit_version"] == "1.0", "incompatible registry version") tasks, families, _ = scoring.registry_data(registry, registry_path.parent) contract = dict(manifest_version="1.0", batch_id=manifest["analysis_id"], seal_record_id="analysis-validation-only", independent_rater_ids=raters, conditions=manifest["conditions"], assignments=[]) _, conditions, _, _ = scoring.manifest_data(contract, tasks) # Adapt only explicitly declared fields to the existing chronology validator. # Its coding/instruction coverage is not used: progress requires snapshots, # not prompt histories. No adapter is published as a run or assessment asset. adapter = dict(repeated_manifest_version="1.0", assessment_id=manifest["analysis_id"], independent_rater_ids=raters, adjudicator_id="progress-validation-only-adjudicator", archives=[], runs=[]) while adapter["adjudicator_id"] in raters: adapter["adjudicator_id"] += "-distinct" for row in sealing.array(manifest["archives"], "archives"): sealing.keys(row, ("archive_id", "path"), "archive descriptor") adapter["archives"].append(dict(archive_id=row["archive_id"], path=str((parent/sealing.text(row["path"], "archive path")).resolve()))) for run in sealing.array(manifest["runs"], "progress runs"): sealing.keys(run, (*RUN_ID, "snapshots"), "progress run") sealing.require(run["condition_id"] in conditions and run["repetition_id"] in conditions[run["condition_id"]], "run outside declared condition/repetition scope") deliveries = [] for snapshot in sealing.array(run["snapshots"], "delivered snapshots"): sealing.keys(snapshot, ("turn", "delivered_at_utc", "artifact", "assessment"), "delivered snapshot") deliveries.append(dict(sequence=snapshot["turn"], delivered_at_utc=snapshot["delivered_at_utc"], artifact=snapshot["artifact"])) adapter["runs"].append(dict({k: run[k] for k in RUN_ID}, trajectory_status="unfinished", trajectory_note="Chronology-only validation adapter; no review/completion assertion", declared_delivery_count=turn_count, deliveries=deliveries, transitions=[])) with tempfile.TemporaryDirectory(prefix="progress-chronology-validation-") as temporary: adapter_path = Path(temporary)/"chronology.json" adapter_path.write_bytes(sealing.json_bytes(adapter)) validated = chronology.load_selection(adapter_path, registry_path) archive_conditions = {} source_runs, source_record_hashes = {}, {} for record in validated["archives"]: inventory, _ = shared.read_json(archived.inside(Path(record["archive"]), "archive-inventory.json"), "sealed inventory") source_runs.update({r["run_id"]: r for r in inventory["runs"]}) record_hashes = {r["record_id"]: r["sha256"] for r in inventory["records"]} source_record_hashes.update({r["run_id"]: record_hashes for r in inventory["runs"]}) source, _ = shared.read_json(archived.inside(Path(record["archive"]), "source-inventory.json"), "source inventory snapshot") for condition in source["conditions"]: archive_conditions.setdefault(condition["condition_id"], set()).update(condition["repetition_ids"]) for condition, repetitions in conditions.items(): sealing.require(condition in archive_conditions and set(repetitions) <= archive_conditions[condition], "analysis condition/repetition not declared by supplied sealed archives") sets, seen_batches = {}, set() for spec in sealing.array(manifest["assessment_sets"], "assessment sets"): sid = sealing.text(spec.get("assessment_set_id"), "assessment set ID") sealing.require(sid not in sets, "duplicate assessment set identity") source = load_assessment_set(spec, parent, registry_path, tasks, raters) batch = (source["source_batch_id"], source["source_seal_record_id"]) sealing.require(batch not in seen_batches, "duplicate source assessment batch identity") seen_batches.add(batch) sets[sid] = source seal_times = {r["source"]["run_id"]: r["sealed_at_utc"] for r in validated["runs"]} snapshots, used_assessments, prescribed = {}, set(), {} for run in manifest["runs"]: rid = run["run_id"] selected_endpoint = source_runs[rid]["endpoint_selection"]["artifact_id"] prescribed[rid] = dict(artifact_record_id=selected_endpoint, sha256=None if selected_endpoint is None else source_record_hashes[rid][selected_endpoint], listed_turn=next((s["turn"] for s in run["snapshots"] if s["artifact"]["record_id"] == selected_endpoint), None), endpoint_changed=False) for snapshot in run["snapshots"]: assessment = snapshot["assessment"] if assessment is not None: sealing.keys(assessment, ("assessment_set_id", "blinded_artifact_id"), "snapshot assessment binding") sid, aid = assessment["assessment_set_id"], assessment["blinded_artifact_id"] sealing.require(sid in sets and aid in sets[sid]["assignments"], "unknown source assessment assignment") sealing.require((sid, aid) not in used_assessments, "duplicate assessment assignment across turns/runs; no carry-forward") used_assessments.add((sid, aid)) assignment = sets[sid]["assignments"][aid] sealing.require(all(assignment[k] == run[k] for k in ("condition_id", "task_id", "task_version", "rubric_version", "repetition_id")) and assignment["family_id"] == tasks[run["task_id"]]["family_id"] and assignment["blinded_artifact_sha256"] == snapshot["artifact"]["sha256"], "assessment/artifact/task/rubric/run-cell identity or hash mismatch") for records in [*sets[sid]["independent"].values(), sets[sid]["adjudicated"]]: record = records.get(aid) if record is not None: sealing.require(scoring.utc(record["completed_at_utc"], "assessment completion") >= scoring.utc(seal_times[rid], "archive batch seal"), "assessment predates corresponding archive batch seal") cell = (run["condition_id"], run["task_id"], run["repetition_id"], snapshot["turn"]) sealing.require(cell not in snapshots, "duplicate run/turn cell") snapshots[cell] = dict(snapshot, run_id=rid) return dict(manifest=manifest, manifest_hash=sealing.sha(content), registry_hash=sealing.sha(registry_path.read_bytes()), tasks=tasks, families=families, conditions=conditions, sets=sets, snapshots=snapshots, prescribed_endpoints=prescribed, archives=validated["archives"], unselected_archived_run_ids=validated["unselected_archived_run_ids"], unused_assessment_assignments=[dict(assessment_set_id=sid, blinded_artifact_id=aid) for sid, source in sets.items() for aid in source["assignments"] if (sid, aid) not in used_assessments]) def mean(values): return math.fsum(values)/len(values) if values else None def aggregate(values, condition, analysis): """Equal repetitions, then tasks within family, then families; no partial means.""" repetitions = analysis["conditions"][condition] tasks, families = analysis["tasks"], analysis["families"] task_means = {task: mean([values[(task, rep)] for rep in repetitions]) if all(values.get((task, rep)) is not None for rep in repetitions) else None for task in tasks} family_means = {family: mean([task_means[t] for t in members]) if all(task_means[t] is not None for t in members) else None for family, members in families.items()} return dict(expected_cells=len(tasks)*len(repetitions), assessed_cells=sum(v is not None for v in values.values()), missing_cells=len(tasks)*len(repetitions)-sum(v is not None for v in values.values()), coverage_complete=all(v is not None for v in family_means.values()), task_means=task_means, family_means=family_means, mean=mean(list(family_means.values())) if all(v is not None for v in family_means.values()) else None) def series_result(analysis, rater=None): expected_runs = {(c, t, rep): None for c, reps in analysis["conditions"].items() for t in analysis["tasks"] for rep in reps} for run in analysis["manifest"]["runs"]: expected_runs[(run["condition_id"], run["task_id"], run["repetition_id"])] = run["run_id"] by_condition = {} for condition, reps in analysis["conditions"].items(): run_rows, point_values = [], {} for task in sorted(analysis["tasks"]): for rep in reps: rid = expected_runs[(condition, task, rep)] turns, previous = [], None for turn in range(1, analysis["manifest"]["turn_count"]+1): snapshot = analysis["snapshots"].get((condition, task, rep, turn)) binding = None if snapshot is None else snapshot["assessment"] record = None if binding is not None: source = analysis["sets"][binding["assessment_set_id"]] lookup = source["adjudicated"] if rater is None else source["independent"][rater] record = lookup.get(binding["blinded_artifact_id"]) score = None if record is None else record["score"] change = score-previous if score is not None and previous is not None else None point_values[(task, rep, turn)] = score turns.append(dict(turn=turn, delivered=snapshot is not None, assessment_assignment_available=binding is not None, assessment_record_available=record is not None, artifact=None if snapshot is None else snapshot["artifact"], assessment_binding=binding, assessment_record=record, normalized_requirement_satisfaction=score, adjacent_change=change, change_from_turn=turn-1 if turn > 1 else None)) previous = score # A missing turn resets adjacency; never bridge it. run_rows.append(dict(run_id=rid, task_id=task, family_id=analysis["tasks"][task]["family_id"], repetition_id=rep, prescribed_endpoint=None if rid is None else analysis["prescribed_endpoints"][rid], turns=turns)) points = [] for turn in range(1, analysis["manifest"]["turn_count"]+1): rows = [run["turns"][turn-1] for run in run_rows] scores = {(t, rep): point_values[(t, rep, turn)] for t in analysis["tasks"] for rep in reps} levels = aggregate(scores, condition, analysis) if turn == 1: changes = dict(applicable=False, expected_cells=0, assessed_cells=0, missing_cells=0, coverage_complete=True, task_means={}, family_means={}, mean=None) else: paired = {(t, rep): (point_values[(t, rep, turn)]-point_values[(t, rep, turn-1)] if point_values[(t, rep, turn)] is not None and point_values[(t, rep, turn-1)] is not None else None) for t in analysis["tasks"] for rep in reps} changes = aggregate(paired, condition, analysis) changes["applicable"] = True observed_criteria = sum(r["assessment_record"]["observed_criterion_judgments"] if r["assessment_record"] else 0 for r in rows) points.append(dict(turn=turn, level=levels, signed_adjacent_change=changes, coverage=dict(expected_run_cells=levels["expected_cells"], missing_run_cells=sum(r["run_id"] is None for r in run_rows), delivered_snapshots=sum(r["delivered"] for r in rows), missing_snapshots=sum(not r["delivered"] for r in rows), missing_assessment_assignments=sum(r["delivered"] and not r["assessment_assignment_available"] for r in rows), missing_assessment_records=sum(r["assessment_assignment_available"] and not r["assessment_record_available"] for r in rows), ineligible_records=sum(r["assessment_record_available"] and r["normalized_requirement_satisfaction"] is None for r in rows), observed_criterion_judgments=observed_criteria, missing_criterion_judgments=12*len(rows)-observed_criteria, expected_criterion_judgments=12*len(rows), complete_score_fraction=levels["assessed_cells"]/levels["expected_cells"] if levels["expected_cells"] else None))) complete = all(p["level"]["coverage_complete"] for p in points) changes_complete = None if len(points) == 1 else all(p["signed_adjacent_change"]["coverage_complete"] for p in points) by_condition[condition] = dict(curve_complete=complete, adjacent_change_curve_complete=changes_complete, turns=points, fully_observed_score_curve=[p["level"]["mean"] for p in points] if complete else None, fully_observed_change_curve=[p["signed_adjacent_change"]["mean"] for p in points[1:]] if changes_complete else None, runs=run_rows) return dict(by_condition=by_condition) def analyze_progress(manifest_path, registry_path=None): analysis = load_analysis(manifest_path, registry_path) return dict(progress_report_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, analysis_id=analysis["manifest"]["analysis_id"], source_manifest_sha256=analysis["manifest_hash"], registry_sha256=analysis["registry_hash"], metric="primary normalized 12-public-requirement satisfaction at declared turns; signed adjacent-turn difference; equal repetition/task/family weighting", endpoint_policy="archived prescribed endpoint retained as metadata; no best-history selection, score carry-forward, or replacement of final-endpoint scoring", archives=analysis["archives"], unselected_archived_run_ids=analysis["unselected_archived_run_ids"], unused_assessment_assignments=analysis["unused_assessment_assignments"], source_assessments={sid: {k: source[k] for k in ("source_batch_id", "source_seal_record_id", "rater_mapping", "provenance")} for sid, source in analysis["sets"].items()}, independent={r: series_result(analysis, r) for r in analysis["manifest"]["independent_rater_ids"]}, adjudicated=series_result(analysis)) def write_report(manifest_path, output_path, registry_path=None): output = packets.output_path(output_path) report = analyze_progress(manifest_path, registry_path) sealing.require(all(output != Path(a["archive"]) and Path(a["archive"]) not in output.parents for a in report["archives"]), "report destination must stay outside sealed archives") content, pending = sealing.json_bytes(report), None try: with tempfile.NamedTemporaryFile(prefix=".pending-progress-", dir=output.parent, delete=False) as handle: pending = Path(handle.name) handle.write(content) handle.flush() os.fsync(handle.fileno()) sealing.require(pending.read_bytes() == content, "report staging byte validation failed") os.link(pending, output) # Atomic publication, no overwrite including races. finally: if pending is not None: try: pending.unlink(missing_ok=True) except OSError: pass # A leftover staging file is not the published report. return dict(notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, output=str(output), sha256=sealing.sha(content)) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--registry", type=Path) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args(argv) try: result = write_report(args.manifest, args.output, args.registry) except (sealing.ValidationError, sealing.account_resources.LedgerError, OSError, KeyError, TypeError, ValueError, ArithmeticError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())