#!/usr/bin/env python3 """Offline first-observed quality-threshold attainment and actual adaptation cost. Development tooling only: no judgments, price inference, interpolation or endpoints changed. """ import argparse from fractions import Fraction import json import math import os from pathlib import Path import sys import tempfile BASE = Path(__file__).resolve().parent sys.path.insert(0, str(BASE.parent/"progress-per-turn")) import analyze_progress as progress shared, archived, sealing, packets = progress.shared, progress.archived, progress.sealing, progress.packets accounting = sealing.account_resources NOTICE = "Development retrospective cost-to-quality tooling, not measured study findings; defaults unfrozen; never adaptation feedback." def read_bound(descriptor, parent, label): sealing.keys(descriptor, ("path", "sha256"), label) path = (parent/sealing.text(descriptor["path"], label+" path")).resolve() return path, packets.hashed_bytes(path, descriptor["sha256"]) def load_analysis(manifest_path, registry_path=None): manifest_path = Path(manifest_path).resolve() content = manifest_path.read_bytes() manifest = sealing.parse_json(content, "cost-threshold manifest", decimal=True) sealing.keys(manifest, ("threshold_manifest_version", "analysis_id", "quality_threshold", "progress_manifest", "ledgers", "run_accounting"), "cost-threshold manifest") sealing.require(manifest["threshold_manifest_version"] == "1.0", "incompatible threshold manifest version") sealing.text(manifest["analysis_id"], "analysis ID") threshold = accounting.decimal_value(manifest["quality_threshold"], "quality threshold") sealing.require(threshold <= 1, "quality threshold must be in [0,1]") progress_path, _ = read_bound(manifest["progress_manifest"], manifest_path.parent, "progress manifest") quality = progress.load_analysis(progress_path, registry_path) sealing.require(quality["manifest_hash"] == manifest["progress_manifest"]["sha256"], "progress manifest changed during validation") runs, attempts, attempt_owners = {}, {}, {} for archive in quality["archives"]: inventory, _ = shared.read_json(archived.inside(Path(archive["archive"]), "archive-inventory.json"), "sealed inventory") for run in inventory["runs"]: runs[run["run_id"]] = run attempts[run["run_id"]] = {a["attempt_id"]: a for a in run["attempts"]} for attempt in run["attempts"]: request_id = attempt["attempt_id"] sealing.require(request_id not in attempt_owners, "duplicate sealed request identity across archives") attempt_owners[request_id] = run["run_id"] ledgers, owners, requests, provenance = {}, {}, {}, [] for descriptor in sealing.array(manifest["ledgers"], "resource ledgers"): sealing.keys(descriptor, ("ledger_id", "path", "sha256"), "resource ledger descriptor") lid = sealing.text(descriptor["ledger_id"], "ledger ID") path, data = read_bound({k: descriptor[k] for k in ("path", "sha256")}, manifest_path.parent, "resource ledger") ledger = sealing.parse_json(data, "resource ledger", decimal=True) ledger_runs, rows = accounting.validate(ledger) sealing.require(lid == ledger["ledger_id"] and lid not in ledgers, "mismatched/duplicate ledger identity") for rid, row in ledger_runs.items(): sealing.require(rid in runs and rid not in owners, "unknown/duplicate ledger run identity; one ledger per run") sealing.require(all(row[k] == runs[rid][k] for k in ("condition_id", "task_id", "repetition_id")), "ledger run identity mismatch") owners[rid] = lid for row in rows: request_id, rid = row["request_id"], row["run_id"] sealing.require(request_id not in requests, "duplicate request identity across ledgers") if request_id in attempt_owners: sealing.require(attempt_owners[request_id] == rid, "request identity belongs to another sealed run") original = attempts[rid][request_id] sealing.require(all(row[k] == original[k] for k in ("phase", "role", "retry_of")), "request/sealed attempt identity mismatch") else: sealing.require(row["phase"] != "adaptation", "adaptation request absent from sealed attempt inventory") # Independent assessment can occur after sealing; its charges are # validated by the ledger contract and never enter adaptation prefixes. requests[request_id] = row for rid, row in ledger_runs.items(): supplied = {r["request_id"] for r in rows if r["run_id"] == rid and r["phase"] == "adaptation"} expected = {a for a, record in attempts[rid].items() if record["phase"] == "adaptation"} if row["phase_log_complete"]["adaptation"]: sealing.require(supplied == expected, "complete adaptation ledger omits sealed attempts") ledgers[lid] = dict(runs=ledger_runs, requests=rows) provenance.append(dict(ledger_id=lid, path=str(path), sha256=descriptor["sha256"])) selected = {r["run_id"]: r for r in quality["manifest"]["runs"]} accounting_runs = {} for row in sealing.array(manifest["run_accounting"], "run accounting"): sealing.keys(row, (*progress.RUN_ID, "ledger_id", "unavailability_reason", "adaptation_request_order", "boundaries"), "run accounting") rid = row["run_id"] sealing.require(rid in selected and rid not in accounting_runs, "unknown/duplicate run accounting identity") sealing.require(all(row[k] == selected[rid][k] for k in progress.RUN_ID), "accounting/progress run identity or version mismatch") lid = row["ledger_id"] if lid is None: sealing.text(row["unavailability_reason"], "unavailable accounting explanation") sealing.require(rid not in owners, "unavailable accounting contradicts a supplied ledger run") else: sealing.require(lid in ledgers and owners.get(rid) == lid and row["unavailability_reason"] is None, "ledger/run binding mismatch") order = row["adaptation_request_order"] sealing.strings(order, "adaptation request order") expected = {a for a, record in attempts[rid].items() if record["phase"] == "adaptation"} sealing.require(len(order) == len(set(order)) and set(order) == expected, "request order must name every sealed adaptation attempt exactly once") positions, last_known_start = {request: i for i, request in enumerate(order)}, None for request_id in order: attempt = attempts[rid][request_id] parent = attempt["retry_of"] sealing.require(parent is None or (parent in positions and positions[parent] < positions[request_id]), "retry precedes its parent request") start = sealing.utc_optional(attempt["started_at_utc"], "attempt start") sealing.require(start is None or last_known_start is None or last_known_start <= start, "reversed request chronology") if start is not None: last_known_start = start snapshots = {s["turn"]: s for s in selected[rid]["snapshots"]} bounds, previous_turn, previous_count = {}, 0, 0 for boundary in sealing.array(row["boundaries"], "snapshot request boundaries"): sealing.keys(boundary, ("turn", "artifact_record_id", "artifact_sha256", "cumulative_request_count", "unavailability_reason"), "snapshot boundary") turn = progress.chronology.integer(boundary["turn"], 1, "boundary turn") sealing.require(turn in snapshots and turn > previous_turn, "duplicate/reversed boundary or boundary for undelivered turn") previous_turn = turn snapshot = snapshots[turn] sealing.require(boundary["artifact_record_id"] == snapshot["artifact"]["record_id"] and boundary["artifact_sha256"] == snapshot["artifact"]["sha256"], "boundary artifact identity/hash mismatch") count = boundary["cumulative_request_count"] if count is None: sealing.text(boundary["unavailability_reason"], "unavailable boundary explanation") else: progress.chronology.integer(count, 0, "cumulative request count") sealing.require(previous_count <= count <= len(order) and boundary["unavailability_reason"] is None, "reversed/out-of-range request boundary") previous_count = count clock = sealing.utc_optional(snapshot["delivered_at_utc"], "snapshot delivery") for i, request_id in enumerate(order): attempt = attempts[rid][request_id] if clock is not None: start = sealing.utc_optional(attempt["started_at_utc"], "attempt start") end = sealing.utc_optional(attempt["ended_at_utc"], "attempt end") if i < count: sealing.require((start is None or start <= clock) and (end is None or end <= clock), "boundary includes work after snapshot or an in-flight request") else: sealing.require(start is None or start >= clock, "boundary excludes a known earlier submission") if attempt["role"] == "worker" and snapshot["artifact"]["record_id"] in attempt["record_ids"]: sealing.require(i < count, "boundary omits an explicitly artifact-linked worker request") bounds[turn] = boundary accounting_runs[rid] = dict(source=row, boundaries=bounds, order=order) sealing.require(set(accounting_runs) == set(selected), "declare available or unavailable accounting for every supplied progress run") return dict(manifest=manifest, manifest_hash=sealing.sha(content), quality=quality, threshold=threshold, ledgers=ledgers, requests=requests, accounting_runs=accounting_runs, ledger_provenance=provenance, unused_ledger_run_ids=sorted(set(owners)-set(selected))) def cumulative_cost(analysis, rid, count): run = analysis["accounting_runs"][rid] lid = run["source"]["ledger_id"] if count is None or lid is None: return dict(coverage_complete=False, request_boundary=count, reason="missing_request_boundary" if count is None else "unavailable_ledger", cost_by_currency=None, resources=None, missing_request_ids=None) ids = run["order"][:count] rows = [analysis["requests"][i] for i in ids if i in analysis["requests"]] sealing.require(all(r["run_id"] == rid and r["phase"] == "adaptation" for r in rows), "adaptation prefix contains a different run or phase") missing = [i for i in ids if i not in analysis["requests"]] log_complete = analysis["ledgers"][lid]["runs"][rid]["phase_log_complete"]["adaptation"] and not missing resources = accounting.summarize_group(rows, log_complete) bill = resources["billing"] known = log_complete and bill["unknown_amount_requests"] == 0 and not bill["unknown_currency_requests"] return dict(coverage_complete=known, request_boundary=count, missing_request_ids=missing, resources=resources, known_zero_request_prefix=known and count == 0, cost_by_currency={c: v["total"] for c, v in bill["by_currency"].items()} if known else None, reason=None if known else "incomplete_adaptation_log_or_billing") def meets_threshold(score, threshold): # The validated kit score is a count / 12. Recover that exact rational to # avoid float-versus-decimal threshold artifacts without altering judgments. count = round(score*12) sealing.require(0 <= count <= 12 and math.isclose(score, count/12, rel_tol=0, abs_tol=1e-12), "incompatible normalized primary score") return Fraction(count, 12) >= Fraction(threshold) def run_result(row, analysis): rid = row["run_id"] quality_complete = all(t["normalized_requirement_satisfaction"] is not None for t in row["turns"]) hit = next((t for t in row["turns"] if t["normalized_requirement_satisfaction"] is not None and meets_threshold(t["normalized_requirement_satisfaction"], analysis["threshold"])), None) snapshots = [] for turn in row["turns"]: bound = None if rid is None else analysis["accounting_runs"][rid]["boundaries"].get(turn["turn"]) count = None if bound is None else bound["cumulative_request_count"] cost = cumulative_cost(analysis, rid, count) if rid is not None else dict(coverage_complete=False, request_boundary=None, reason="missing_run", cost_by_currency=None, resources=None, missing_request_ids=None) snapshots.append(dict(turn=turn["turn"], delivered=turn["delivered"], score=turn["normalized_requirement_satisfaction"], artifact=turn["artifact"], threshold_met=None if turn["normalized_requirement_satisfaction"] is None else meets_threshold(turn["normalized_requirement_satisfaction"], analysis["threshold"]), cumulative_adaptation=cost)) total = None if rid is None else cumulative_cost(analysis, rid, len(analysis["accounting_runs"][rid]["order"])) accounting_complete = total is not None and total["coverage_complete"] and all(s["cumulative_adaptation"]["coverage_complete"] for s in snapshots) first_cost = None if hit is None else snapshots[hit["turn"]-1]["cumulative_adaptation"] quality_outcome = "observed_attainment" if hit else ("fully_observed_non_attainment" if quality_complete else "indeterminate") outcome = ("observed_attainment" if first_cost["coverage_complete"] else "indeterminate") if hit else ( "fully_observed_non_attainment" if quality_complete and accounting_complete else "indeterminate") final_assessment = None if rid is not None: lid = analysis["accounting_runs"][rid]["source"]["ledger_id"] if lid is not None: ledger = analysis["ledgers"][lid] final_assessment = accounting.summarize_group([r for r in ledger["requests"] if r["run_id"] == rid and r["phase"] == "final_assessment"], ledger["runs"][rid]["phase_log_complete"]["final_assessment"]) return dict(run_id=rid, task_id=row["task_id"], family_id=row["family_id"], repetition_id=row["repetition_id"], prescribed_endpoint=row["prescribed_endpoint"], quality_outcome=quality_outcome, threshold_analysis_outcome=outcome, quality_coverage_complete=quality_complete, accounting_coverage_complete=accounting_complete, trajectory_complete=quality_complete and accounting_complete, missing_assessment_turns=[s["turn"] for s in snapshots if s["score"] is None], missing_accounting_turns=[s["turn"] for s in snapshots if not s["cumulative_adaptation"]["coverage_complete"]], first_observed_attainment=None if hit is None else dict(turn=hit["turn"], score=hit["normalized_requirement_satisfaction"], artifact=hit["artifact"], no_missing_assessments_before_hit=all(t["normalized_requirement_satisfaction"] is not None for t in row["turns"][:hit["turn"]-1]), cumulative_adaptation=first_cost, actual_billed_cost_by_currency=first_cost["cost_by_currency"]), full_declared_adaptation=total, final_assessment_resources_separate=final_assessment, snapshots=snapshots) def series_result(quality_series, analysis): result = {} for condition, source in quality_series["by_condition"].items(): rows = [run_result(row, analysis) for row in source["runs"]] count = len(rows) complete = all(r["trajectory_complete"] for r in rows) quality_complete = all(r["quality_coverage_complete"] for r in rows) outcomes = {s: sum(r["threshold_analysis_outcome"] == s for r in rows) for s in ("observed_attainment", "fully_observed_non_attainment", "indeterminate")} quality_counts = {s: sum(r["quality_outcome"] == s for r in rows) for s in outcomes} result[condition] = dict(expected_run_denominator=count, observed_run_count=sum(r["run_id"] is not None for r in rows), complete_trajectory_count=sum(r["trajectory_complete"] for r in rows), incomplete_trajectory_count=sum(not r["trajectory_complete"] for r in rows), threshold_analysis_outcome_counts=outcomes, quality_outcome_counts=quality_counts, complete_cohort_coverage=complete, observed_attainment_rate=outcomes["observed_attainment"]/count if complete and count else None, quality_attainment_rate=quality_counts["observed_attainment"]/count if quality_complete and count else None, cost_aggregation="per-run actual currency vectors only; no pooled, available-case, or censored population cost mean", runs=rows) return dict(by_condition=result) def analyze_threshold(manifest_path, registry_path=None): analysis = load_analysis(manifest_path, registry_path) quality = analysis["quality"] return dict(threshold_report_version="1.0", notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, analysis_id=analysis["manifest"]["analysis_id"], source_manifest_sha256=analysis["manifest_hash"], progress_manifest_sha256=quality["manifest_hash"], registry_sha256=quality["registry_hash"], quality_threshold=format(analysis["threshold"], "f"), threshold_rule="normalized primary requirement satisfaction >= declared threshold, compared as exact count/12 rationals", estimand="first assessed delivered snapshot meeting threshold, at its explicit cumulative adaptation-request boundary; not persistent or continuous-time attainment", archives=quality["archives"], source_assessments={sid: {k: s[k] for k in ("source_batch_id", "source_seal_record_id", "rater_mapping", "provenance")} for sid, s in quality["sets"].items()}, ledger_provenance=analysis["ledger_provenance"], unused_ledger_run_ids=analysis["unused_ledger_run_ids"], unselected_archived_run_ids=quality["unselected_archived_run_ids"], unused_assessment_assignments=quality["unused_assessment_assignments"], independent={r: series_result(progress.series_result(quality, r), analysis) for r in quality["manifest"]["independent_rater_ids"]}, adjudicated=series_result(progress.series_result(quality), analysis)) def write_report(manifest_path, output_path, registry_path=None): output = packets.output_path(output_path) report = analyze_threshold(manifest_path, registry_path) sealing.require(all(output != Path(a["archive"]) and Path(a["archive"]) not in output.parents for a in report["archives"]), "report destination must stay outside sealed archives") content, pending = sealing.json_bytes(report), None try: with tempfile.NamedTemporaryFile(prefix=".pending-threshold-", dir=output.parent, delete=False) as handle: pending = Path(handle.name) handle.write(content) handle.flush() os.fsync(handle.fileno()) sealing.require(pending.read_bytes() == content, "report staging byte validation failed") os.link(pending, output) finally: if pending is not None: try: pending.unlink(missing_ok=True) except OSError: pass return dict(notice=NOTICE, coordinator_only=True, defaults_unfrozen=True, output=str(output), sha256=sealing.sha(content)) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--registry", type=Path) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args(argv) try: result = write_report(args.manifest, args.output, args.registry) except (sealing.ValidationError, accounting.LedgerError, OSError, KeyError, TypeError, ValueError, ArithmeticError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())