"""Artificial accounting fixtures only; no model calls or measured study costs.""" import copy from decimal import Decimal import json from pathlib import Path import subprocess import sys import tempfile import unittest import account_resources as accounting class AccountingTests(unittest.TestCase): def setUp(self): self.ledger = dict(ledger_version="1.0", ledger_id="artificial-accounting-fixture", runs=[self.run_record()], requests=[]) def run_record(self, rid="fixture-run", condition="fixture-condition", rep="fixture-repeat"): return dict(run_id=rid, condition_id=condition, task_id="fixture-task-not-a-study-answer", repetition_id=rep, phase_log_complete=dict(adaptation=True, final_assessment=True)) def request(self, rid="fixture-request", run="fixture-run", role="worker", phase="adaptation", amount=0.1, currency="USD"): return dict(request_id=rid, run_id=run, phase=phase, role=role, status="succeeded", disposition="retained", retry_of=None, provider="fictional-fixture-provider", model_snapshot=None, pricing_date=None, usage=dict(input_tokens=100, cached_input_tokens=40, cached_input_relationship="included", output_tokens=30, reasoning_tokens=10, reasoning_relationship="included", wall_time_seconds=1.25, tool_runtime_seconds=0), billed_cost=dict(amount=amount, currency=currency)) def summary(self): return accounting.summarize_ledger(self.ledger) def adaptation(self): return self.summary()["all_runs"]["adaptation"]["totals"] def test_subsets_are_not_double_counted(self): self.ledger["requests"] = [self.request()] total = self.adaptation() self.assertEqual(total["normalized_tokens"]["nonoverlapping_total"]["total"], 130) self.assertEqual(total["reported_tokens"]["cached_input_tokens"]["total"], 40) self.assertEqual(total["reported_tokens"]["reasoning_tokens"]["total"], 10) def test_additional_and_unknown_relationships(self): row = self.request() self.ledger["requests"] = [row] row["usage"].update(cached_input_relationship="additional", reasoning_relationship="additional") self.assertEqual(self.adaptation()["normalized_tokens"]["nonoverlapping_total"]["total"], 180) row["usage"]["cached_input_relationship"] = "unknown" self.assertIsNone(self.adaptation()["normalized_tokens"]["nonoverlapping_total"]["total"]) self.assertEqual(self.adaptation()["reported_tokens"]["input_tokens"]["total"], 100) def test_unknown_subset_does_not_erase_inclusive_parent(self): row = self.request() self.ledger["requests"] = [row] row["usage"]["cached_input_tokens"] = None total = self.adaptation() self.assertEqual(total["normalized_tokens"]["input_including_cached"]["total"], 100) self.assertIsNone(total["reported_tokens"]["cached_input_tokens"]["total"]) row["usage"]["cached_input_relationship"] = "additional" self.assertIsNone(self.adaptation()["normalized_tokens"]["input_including_cached"]["total"]) def test_missing_usage_and_billing_remain_unknown(self): row = self.request(amount=None, currency=None) for field in accounting.TOKEN_FIELDS: row["usage"][field] = None row["usage"]["wall_time_seconds"] = None self.ledger["requests"] = [row] total = self.adaptation() metric = total["reported_tokens"]["input_tokens"] self.assertEqual((metric["known_records"], metric["unknown_records"]), (0, 1)) self.assertIsNone(metric["total"]) self.assertIsNone(metric["known_subtotal"]) self.assertIsNone(total["billing"]["single_currency_billed_total"]) self.assertEqual(total["billing"]["unknown_amount_requests"], 1) def test_partial_billing_known_subtotal_is_not_total(self): self.ledger["requests"] = [self.request(), self.request("fixture-unknown-charge", amount=None)] metric = self.adaptation()["billing"]["by_currency"]["USD"] self.assertEqual(metric["known_subtotal"], "0.1") self.assertEqual(metric["unknown_records"], 1) self.assertIsNone(metric["total"]) self.assertIsNone(self.adaptation()["billing"]["single_currency_billed_total"]) def test_currencies_are_never_converted_or_combined(self): self.ledger["requests"] = [self.request(), self.request("fixture-euro", amount=0.2, currency="EUR")] bill = self.adaptation()["billing"] self.assertEqual(bill["by_currency"]["USD"]["total"], "0.1") self.assertEqual(bill["by_currency"]["EUR"]["total"], "0.2") self.assertTrue(bill["mixed_reported_currencies"]) self.assertIsNone(bill["single_currency_billed_total"]) self.ledger["requests"].append(self.request("fixture-unknown-currency", amount=0.3, currency=None)) bill = self.adaptation()["billing"] self.assertIsNone(bill["by_currency"]["USD"]["total"]) self.assertEqual(bill["unknown_currency_requests"][0]["reported_amount"], "0.3") def test_rejected_failed_cancelled_comparisons_and_retries_all_count(self): original = self.request("fixture-failed", role="comparator", amount=0.1) original.update(status="failed", disposition="discarded") retry = self.request("fixture-retry", role="comparator", amount=0.2) retry.update(retry_of=original["request_id"], disposition="rejected") cancelled = self.request("fixture-cancelled", role="worker", amount=0) cancelled.update(status="cancelled", disposition="unknown") self.ledger["requests"] = [retry, original, cancelled] total = self.adaptation() self.assertEqual(total["recorded_request_count"], 3) self.assertEqual(total["retry_attempt_count"], 1) self.assertEqual(total["normalized_tokens"]["nonoverlapping_total"]["total"], 390) self.assertEqual(total["billing"]["single_currency_billed_total"]["amount"], "0.3") self.assertEqual(total["status_counts"]["failed"], 1) self.assertEqual(total["disposition_counts"]["rejected"], 1) def test_final_assessment_is_separate_from_adaptation(self): self.ledger["requests"] = [self.request(), self.request("fixture-final", role="scorer", phase="final_assessment", amount=0.4)] result = self.summary()["all_runs"] self.assertEqual(result["adaptation"]["totals"]["recorded_request_count"], 1) self.assertEqual(result["final_assessment"]["totals"]["recorded_request_count"], 1) self.assertEqual(result["adaptation"]["totals"]["billing"]["single_currency_billed_total"]["amount"], "0.1") self.assertEqual(result["final_assessment"]["totals"]["billing"]["single_currency_billed_total"]["amount"], "0.4") def test_run_condition_role_grouping_conserves_requests(self): self.ledger["runs"] += [self.run_record("fixture-run-2", rep="fixture-repeat-2"), self.run_record("fixture-run-3", condition="fixture-other")] self.ledger["requests"] = [self.request(), self.request("fixture-2", "fixture-run-2", role="critic"), self.request("fixture-3", "fixture-run-3", role="optimizer")] result = self.summary() self.assertEqual(result["by_condition"]["fixture-condition"]["adaptation"]["totals"]["recorded_request_count"], 2) self.assertEqual(sum(r["recorded_request_count"] for r in result["all_runs"]["adaptation"]["roles"].values()), 3) self.assertEqual(result["by_run"]["fixture-run-2"]["phases"]["adaptation"]["roles"]["critic"]["recorded_request_count"], 1) def test_incomplete_logs_withhold_totals_and_known_empty_is_not_missing(self): self.ledger["requests"] = [self.request()] self.ledger["runs"][0]["phase_log_complete"]["adaptation"] = False total = self.adaptation() self.assertFalse(total["request_count_complete"]) self.assertEqual(total["normalized_tokens"]["nonoverlapping_total"]["known_subtotal"], 130) self.assertIsNone(total["normalized_tokens"]["nonoverlapping_total"]["total"]) self.assertIsNone(total["billing"]["by_currency"]["USD"]["total"]) self.ledger["requests"] = [] self.ledger["runs"][0]["phase_log_complete"]["adaptation"] = True self.assertEqual(self.adaptation()["normalized_tokens"]["nonoverlapping_total"]["total"], 0) self.assertIsNone(self.adaptation()["billing"]["single_currency_billed_total"]) def test_decimal_billing_is_exact_without_inferred_prices(self): self.ledger["requests"] = [self.request(amount=Decimal("10000000000000000000000000000.01")), self.request("fixture-small", amount=Decimal("0.02"))] total = self.adaptation()["billing"]["single_currency_billed_total"]["amount"] self.assertEqual(total, "10000000000000000000000000000.03") def test_invalid_numeric_values_and_subset_claims(self): for field, bad in [("input_tokens", True), ("input_tokens", 1.0), ("input_tokens", -1), ("reasoning_tokens", "2"), ("wall_time_seconds", -0.1), ("tool_runtime_seconds", float("nan"))]: self.ledger["requests"] = [self.request()] self.ledger["requests"][0]["usage"][field] = bad with self.subTest(field=field, value=bad), self.assertRaises(accounting.LedgerError): self.summary() for bad in [True, -1, float("inf"), "0.1"]: self.ledger["requests"] = [self.request(amount=bad)] with self.subTest(amount=bad), self.assertRaises(accounting.LedgerError): self.summary() for field, value in [("cached_input_tokens", 101), ("reasoning_tokens", 31)]: self.ledger["requests"] = [self.request()] self.ledger["requests"][0]["usage"][field] = value with self.assertRaises(accounting.LedgerError): self.summary() def test_duplicate_records_run_identities_and_undeclared_runs(self): self.ledger["requests"] = [self.request(), self.request()] with self.assertRaisesRegex(accounting.LedgerError, "duplicate request"): self.summary() self.ledger["requests"] = [self.request(run="absent")] with self.assertRaisesRegex(accounting.LedgerError, "undeclared run"): self.summary() self.ledger["requests"] = [] self.ledger["runs"].append(self.run_record("new-ID-same-logical-run")) with self.assertRaisesRegex(accounting.LedgerError, "duplicate run"): self.summary() def test_retry_missing_cycle_and_cross_run_rejected(self): row = self.request() self.ledger["requests"] = [row] for parent in ["absent", row["request_id"]]: row["retry_of"] = parent with self.assertRaisesRegex(accounting.LedgerError, "retry_of"): self.summary() row["retry_of"] = None retry = self.request("fixture-retry") retry["retry_of"] = row["request_id"] row["retry_of"] = retry["request_id"] self.ledger["requests"].append(retry) with self.assertRaisesRegex(accounting.LedgerError, "cyclic"): self.summary() row["retry_of"] = None self.ledger["runs"].append(self.run_record("fixture-different-run", rep="other-repeat")) retry["run_id"] = "fixture-different-run" with self.assertRaisesRegex(accounting.LedgerError, "retry must retain"): self.summary() def test_invalid_versions_currency_and_dates(self): self.ledger["ledger_version"] = "2.0" with self.assertRaises(accounting.LedgerError): self.summary() self.ledger["ledger_version"] = "1.0" self.ledger["requests"] = [self.request(currency="usd")] with self.assertRaises(accounting.LedgerError): self.summary() self.ledger["requests"] = [self.request()] self.ledger["requests"][0]["pricing_date"] = "2000-02-31" with self.assertRaises(accounting.LedgerError): self.summary() def test_json_cli_read_only_unknowns_and_duplicate_keys(self): self.ledger["requests"] = [self.request(amount=None)] original_object = copy.deepcopy(self.ledger) self.summary() self.assertEqual(self.ledger, original_object) with tempfile.TemporaryDirectory(prefix="artificial-resource-fixture-") as tmp: path = Path(tmp)/"artificial-ledger.json" path.write_text(json.dumps(self.ledger)) before = path.read_bytes() command = [sys.executable, "-B", str(Path(accounting.__file__).resolve()), "--ledger", str(path)] run = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(run.returncode, 0, run.stderr) self.assertEqual(path.read_bytes(), before) bill = json.loads(run.stdout)["all_runs"]["adaptation"]["totals"]["billing"] self.assertIsNone(bill["single_currency_billed_total"]) path.write_text('{"ledger_version":"1.0", "ledger_version":"2.0"}') bad = subprocess.run(command, capture_output=True, text=True, check=False) self.assertEqual(bad.returncode, 2) self.assertEqual(bad.stdout, "") self.assertIn("duplicate JSON key", json.loads(bad.stderr)["error"]) if __name__ == "__main__": unittest.main()