#!/usr/bin/env python3 """Read-only resource accounting for a versioned request ledger (standard library). Development tooling, not measured study costs. No price estimation or model calls. """ import argparse from collections import Counter from decimal import Decimal, localcontext import json import math from pathlib import Path import re import sys NOTICE = "Development accounting tooling, not measured study costs; summaries depend on the supplied ledger." PHASES = ("adaptation", "final_assessment") ROLES = ("worker", "planner", "optimizer", "critic", "comparator", "selector", "scorer", "artifact_rendering", "tool", "other") RELATIONS = ("included", "additional", "unknown") STATUSES = ("succeeded", "failed", "cancelled", "unknown") DISPOSITIONS = ("retained", "rejected", "discarded", "not_applicable", "unknown") TOKEN_FIELDS = ("input_tokens", "cached_input_tokens", "output_tokens", "reasoning_tokens") USAGE_FIELDS = (*TOKEN_FIELDS, "cached_input_relationship", "reasoning_relationship", "wall_time_seconds", "tool_runtime_seconds") class LedgerError(ValueError): pass def require(condition, message): if not condition: raise LedgerError(message) def exact_keys(value, expected, label): require(isinstance(value, dict) and set(value) == set(expected), f"{label}: missing or unexpected fields") def text(value, label): require(isinstance(value, str) and bool(value.strip()), f"{label}: expected nonempty string") return value def optional_text(value, label): if value is not None: text(value, label) def decimal_value(value, label): require(type(value) in (int, float, Decimal), f"{label}: expected numeric value or null") if type(value) is float: require(math.isfinite(value), f"{label}: non-finite number") result = Decimal(str(value)) require(result.is_finite() and result >= 0, f"{label}: expected finite nonnegative value") return result def decimal_sum(values): """Exact sum of finite nonnegative decimal inputs, independent of default precision.""" if not values: return Decimal(0) scale = max(max(0, -v.as_tuple().exponent) for v in values) places = max(max(1, v.adjusted()+1) for v in values) with localcontext() as context: context.prec = scale + places + len(str(len(values))) + 2 return sum(values, Decimal(0)) def read_ledger(path): def unique(pairs): result = {} for key, value in pairs: require(key not in result, f"duplicate JSON key: {key}") result[key] = value return result def bad_constant(value): raise LedgerError(f"non-finite JSON number: {value}") try: return json.loads(Path(path).read_text(encoding="utf-8"), object_pairs_hook=unique, parse_float=Decimal, parse_constant=bad_constant) except (OSError, json.JSONDecodeError) as exc: raise LedgerError(str(exc)) from exc def normalize_side(parent, subset, relationship): if relationship == "included": return parent # A missing subset does not erase a reported inclusive total. if relationship == "additional" and parent is not None and subset is not None: return parent + subset return None def validate(ledger): exact_keys(ledger, ("ledger_version", "ledger_id", "runs", "requests"), "ledger") require(ledger["ledger_version"] == "1.0", "unsupported ledger_version; expected 1.0") text(ledger["ledger_id"], "ledger_id") require(isinstance(ledger["runs"], list) and isinstance(ledger["requests"], list), "runs and requests must be arrays") runs, identities, requests = {}, set(), {} for row in ledger["runs"]: exact_keys(row, ("run_id", "condition_id", "task_id", "repetition_id", "phase_log_complete"), "run") for field in ("run_id", "condition_id", "task_id", "repetition_id"): text(row[field], field) identity = tuple(row[k] for k in ("condition_id", "task_id", "repetition_id")) require(row["run_id"] not in runs and identity not in identities, "duplicate run ID or condition/task/repetition identity") exact_keys(row["phase_log_complete"], PHASES, "phase_log_complete") require(all(type(v) is bool for v in row["phase_log_complete"].values()), "phase completeness must be explicit booleans") runs[row["run_id"]] = row identities.add(identity) require(runs, "declare at least one run") for row in ledger["requests"]: exact_keys(row, ("request_id", "run_id", "phase", "role", "status", "disposition", "retry_of", "provider", "model_snapshot", "pricing_date", "usage", "billed_cost"), "request") rid = text(row["request_id"], "request_id") require(rid not in requests, "duplicate request_id; each attempt must be recorded once") require(row["run_id"] in runs, "request references undeclared run") require(row["phase"] in PHASES and row["role"] in ROLES, "unknown phase or role") require(row["status"] in STATUSES and row["disposition"] in DISPOSITIONS, "unknown status or disposition") for field in ("provider", "model_snapshot"): optional_text(row[field], field) date = row["pricing_date"] if date is not None: text(date, "pricing_date") require(re.fullmatch(r"\d{4}-\d{2}-\d{2}", date) is not None, "pricing_date must use YYYY-MM-DD") from datetime import date as calendar_date try: calendar_date.fromisoformat(date) except ValueError as exc: raise LedgerError("invalid pricing_date") from exc optional_text(row["retry_of"], "retry_of") usage = row["usage"] exact_keys(usage, USAGE_FIELDS, "usage") for field in TOKEN_FIELDS: value = usage[field] require(value is None or (type(value) is int and value >= 0), f"{field}: expected nonnegative integer or null") for field in ("cached_input_relationship", "reasoning_relationship"): require(usage[field] in RELATIONS, f"{field}: unknown relationship") for parent, subset, relationship in (("input_tokens", "cached_input_tokens", "cached_input_relationship"), ("output_tokens", "reasoning_tokens", "reasoning_relationship")): if usage[relationship] == "included" and usage[parent] is not None and usage[subset] is not None: require(usage[subset] <= usage[parent], f"{subset} exceeds inclusive {parent}") for field in ("wall_time_seconds", "tool_runtime_seconds"): if usage[field] is not None: decimal_value(usage[field], field) bill = row["billed_cost"] exact_keys(bill, ("amount", "currency"), "billed_cost") if bill["amount"] is not None: decimal_value(bill["amount"], "billed amount") if bill["currency"] is not None: require(isinstance(bill["currency"], str) and re.fullmatch(r"[A-Z]{3}", bill["currency"]) is not None, "currency must be an explicit uppercase three-letter code or null") requests[rid] = row # Validate links after loading: input record order is immaterial. for rid, row in requests.items(): seen = {rid} parent_id = row["retry_of"] while parent_id is not None: require(parent_id in requests and parent_id not in seen, "missing or cyclic retry_of reference") parent = requests[parent_id] require(all(parent[k] == row[k] for k in ("run_id", "phase", "role")), "retry must retain run, phase, and role") seen.add(parent_id) parent_id = parent["retry_of"] return runs, list(requests.values()) def metric(values, log_complete, decimal=False): known = [v for v in values if v is not None] total = decimal_sum(known) if decimal else sum(known) serialize = (lambda v: format(v, "f")) if decimal else (lambda v: v) return dict(known_records=len(known), unknown_records=len(values)-len(known), known_subtotal=serialize(total) if known or not values else None, total=serialize(total) if log_complete and len(known) == len(values) else None) def summarize_group(rows, log_complete): reported = {field: [] for field in TOKEN_FIELDS} normalized = {field: [] for field in ("input_including_cached", "output_including_reasoning", "nonoverlapping_total")} durations = {field: [] for field in ("wall_time_seconds", "tool_runtime_seconds")} bills, unknown_currency = {}, [] for row in rows: usage = row["usage"] for field in reported: reported[field].append(usage[field]) input_total = normalize_side(usage["input_tokens"], usage["cached_input_tokens"], usage["cached_input_relationship"]) output_total = normalize_side(usage["output_tokens"], usage["reasoning_tokens"], usage["reasoning_relationship"]) normalized["input_including_cached"].append(input_total) normalized["output_including_reasoning"].append(output_total) normalized["nonoverlapping_total"].append(None if input_total is None or output_total is None else input_total+output_total) for field in durations: durations[field].append(None if usage[field] is None else decimal_value(usage[field], field)) bill = row["billed_cost"] amount = None if bill["amount"] is None else decimal_value(bill["amount"], "billed amount") if bill["currency"] is None: unknown_currency.append(dict(request_id=row["request_id"], reported_amount=None if amount is None else format(amount, "f"))) else: bills.setdefault(bill["currency"], []).append(amount) by_currency = {c: metric(bills[c], log_complete and not unknown_currency, decimal=True) for c in sorted(bills)} single_total = None if len(by_currency) == 1 and not unknown_currency: currency = next(iter(by_currency)) if by_currency[currency]["total"] is not None: single_total = dict(currency=currency, amount=by_currency[currency]["total"]) return dict(recorded_request_count=len(rows), request_count_complete=log_complete, status_counts=dict(sorted(Counter(r["status"] for r in rows).items())), disposition_counts=dict(sorted(Counter(r["disposition"] for r in rows).items())), retry_attempt_count=sum(r["retry_of"] is not None for r in rows), reported_tokens={f: metric(values, log_complete) for f, values in reported.items()}, normalized_tokens={f: metric(values, log_complete) for f, values in normalized.items()}, relationship_counts={f: dict(sorted(Counter(r["usage"][f] for r in rows).items())) for f in ("cached_input_relationship", "reasoning_relationship")}, durations_seconds={f: metric(values, log_complete, decimal=True) for f, values in durations.items()}, billing=dict(by_currency=by_currency, single_currency_billed_total=single_total, mixed_reported_currencies=len(by_currency) > 1, unknown_amount_requests=sum(r["billed_cost"]["amount"] is None for r in rows), unknown_currency_requests=unknown_currency)) def summarize_ledger(ledger): runs, requests = validate(ledger) def scope(run_ids): result = {} for phase in PHASES: selected = [r for r in requests if r["run_id"] in run_ids and r["phase"] == phase] complete = all(runs[r]["phase_log_complete"][phase] for r in run_ids) result[phase] = dict(totals=summarize_group(selected, complete), roles={role: summarize_group([r for r in selected if r["role"] == role], complete) for role in ROLES}) return result by_run = {rid: dict(identity={k: row[k] for k in ("run_id", "condition_id", "task_id", "repetition_id")}, phases=scope({rid})) for rid, row in sorted(runs.items())} conditions = sorted({r["condition_id"] for r in runs.values()}) return dict(accounting_report_version="1.0", ledger_version="1.0", ledger_id=ledger["ledger_id"], notice=NOTICE, coordinator_only=True, by_run=by_run, by_condition={c: scope({rid for rid, run in runs.items() if run["condition_id"] == c}) for c in conditions}, all_runs=scope(set(runs))) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--ledger", type=Path, required=True) args = parser.parse_args(argv) try: result = summarize_ledger(read_ledger(args.ledger)) except (LedgerError, OSError, KeyError, TypeError, ValueError) as exc: print(json.dumps(dict(error=str(exc), notice=NOTICE)), file=sys.stderr) return 2 print(json.dumps(result, indent=2, allow_nan=False)) return 0 if __name__ == "__main__": sys.exit(main())