"""Temporary artificial numeric/metadata/accounting fixtures; no study findings.""" import contextlib import copy from decimal import Decimal import io import json from pathlib import Path import sys import unittest from unittest import mock import analyze_cost_threshold as threshold sys.path.insert(0, str(Path(__file__).resolve().parent.parent/"progress-per-turn")) sys.path.insert(0, str(Path(__file__).resolve().parent.parent/"resource-accounting")) import test_analyze_progress as quality_fixtures import test_account_resources as accounting_fixtures class ThresholdTests(unittest.TestCase): def setUp(self): self.quality = quality_fixtures.ProgressTests() self.quality.setUp() self.addCleanup(self.quality.doCleanups) f = self.quality.archive self.ledger = dict(ledger_version="1.0", ledger_id="artificial-threshold-ledger", runs=[], requests=[]) bill_fixture = accounting_fixtures.AccountingTests() bill_fixture.setUp() self.orders = {} for run in f.source["runs"]: rid = run["run_id"] attempts = [] self.ledger["runs"].append(dict(run_id=rid, condition_id=run["condition_id"], task_id=run["task_id"], repetition_id=run["repetition_id"], phase_log_complete=dict(adaptation=True, final_assessment=True))) # Include discarded proposal, failed comparison, retry, and cancelled # tail work. This is artificial accounting data, not measured costs. roles = ["worker", "optimizer", "comparator", "comparator", "worker", "planner", "worker"] starts = ["00.1", "01.1", "01.3", "01.5", "01.7", "02.1", "02.3"] ends = ["00.2", "01.2", "01.4", "01.6", "01.8", "02.2", "02.4"] for index, role in enumerate(roles, 1): request_id = f"artificial-request-{rid}-{index}" parent = f"artificial-request-{rid}-3" if index == 4 else None status = "failed" if index == 3 else ("cancelled" if index == 6 else "succeeded") record_ids = ["fixture-prompt-a"] if index in (1, 5, 7): turn = {1: 1, 5: 2, 7: 3}[index] record_ids.append(self.quality.artifacts[(run["task_id"], run["repetition_id"], turn)]["record_id"]) attempts.append(dict(attempt_id=request_id, phase="adaptation", role=role, status=status, retry_of=parent, started_at_utc="2000-01-01T00:00:"+starts[index-1]+"Z", ended_at_utc="2000-01-01T00:00:"+ends[index-1]+"Z", record_ids=record_ids)) request = bill_fixture.request(request_id, rid, role=role, amount=index/10) request.update(status=status, retry_of=parent, disposition="discarded" if index in (2, 6) else ("rejected" if index in (3, 4) else "retained")) self.ledger["requests"].append(request) self.orders[rid] = [a["attempt_id"] for a in attempts] run["attempts"] = attempts self.ledger["requests"].append(bill_fixture.request("independent-assessment-"+rid, rid, role="scorer", phase="final_assessment", amount=99)) # Re-seal only temporary artificial fixtures, before any test assessments. f.output = f.root/"threshold-artificial-archive" f.seal() seal = json.loads((f.output/"packet-build-manifest.json").read_bytes())["seal_record_id"] self.quality.manifest["archives"][0]["path"] = str(f.output) for run in self.quality.manifest["runs"]: run["seal_record_id"] = seal for fixture in self.quality.fixtures.values(): for form in fixture.forms: for row in form["ratings"]: row["completed_at_utc"] = self.quality.now() form["locked_at_utc"] = self.quality.now() self.ledger_path = f.root/"artificial-resource-ledger.json" self.manifest_path = f.root/"threshold-selection.json" self.output = f.root/"new-threshold-report.json" self.manifest = dict(threshold_manifest_version="1.0", analysis_id="artificial-cost-threshold-analysis", quality_threshold=.5, progress_manifest={}, ledgers=[], run_accounting=[]) for run in self.quality.manifest["runs"]: row = {k: run[k] for k in threshold.progress.RUN_ID} row.update(ledger_id=self.ledger["ledger_id"], unavailability_reason=None, adaptation_request_order=self.orders[run["run_id"]], boundaries=[]) for snapshot, count in zip(run["snapshots"], (1, 5, 7)): row["boundaries"].append(dict(turn=snapshot["turn"], artifact_record_id=snapshot["artifact"]["record_id"], artifact_sha256=snapshot["artifact"]["sha256"], cumulative_request_count=count, unavailability_reason=None)) self.manifest["run_accounting"].append(row) @property def rid(self): return self.manifest["run_accounting"][0]["run_id"] def prepare(self): self.quality.prepare() self.ledger_path.write_bytes(threshold.sealing.json_bytes(self.ledger)) self.manifest["progress_manifest"] = self.quality.descriptor(self.quality.manifest_path) self.manifest["ledgers"] = [dict(ledger_id=self.ledger["ledger_id"], **self.quality.descriptor(self.ledger_path))] self.save_manifest() def save_manifest(self): self.manifest_path.write_bytes(threshold.sealing.json_bytes(self.manifest)) def analyze(self): self.prepare() return threshold.analyze_threshold(self.manifest_path, self.quality.archive.registry_path) def group(self, report, rater="analysis-rater-a"): return report["independent"][rater]["by_condition"][self.quality.condition] def target(self, report, rater="analysis-rater-a"): return next(r for r in self.group(report, rater)["runs"] if r["run_id"] == self.rid) def test_first_observed_hit_nonmonotonic_scores_and_all_attempt_costs(self): report = self.analyze() row = self.target(report) hit = row["first_observed_attainment"] self.assertEqual(hit["turn"], 2) self.assertEqual(hit["actual_billed_cost_by_currency"], {"USD": "1.5"}) resources = hit["cumulative_adaptation"]["resources"] self.assertEqual(resources["recorded_request_count"], 5) self.assertEqual(resources["retry_attempt_count"], 1) self.assertEqual(resources["status_counts"]["failed"], 1) self.assertEqual(resources["disposition_counts"]["discarded"], 1) self.assertEqual(resources["disposition_counts"]["rejected"], 2) self.assertEqual(row["full_declared_adaptation"]["cost_by_currency"], {"USD": "2.8"}) self.assertEqual(row["final_assessment_resources_separate"]["billing"]["single_currency_billed_total"]["amount"], "99") self.assertFalse(row["snapshots"][2]["threshold_met"]) self.assertEqual(row["prescribed_endpoint"]["listed_turn"], 3) def test_rater_and_adjudication_series_not_pooled(self): report = self.analyze() self.assertEqual(self.target(report)["first_observed_attainment"]["turn"], 2) self.assertEqual(self.target(report, "analysis-rater-b")["first_observed_attainment"]["turn"], 1) adjudicated = report["adjudicated"]["by_condition"][self.quality.condition] self.assertEqual(adjudicated["runs"][0]["first_observed_attainment"]["actual_billed_cost_by_currency"], {"USD": "0.1"}) def test_fully_observed_non_attainment_has_no_assigned_event_cost(self): self.manifest["quality_threshold"] = .75 group = self.group(self.analyze()) self.assertEqual(group["threshold_analysis_outcome_counts"]["fully_observed_non_attainment"], 24) self.assertEqual(group["observed_attainment_rate"], 0) self.assertTrue(group["complete_cohort_coverage"]) self.assertIsNone(group["runs"][0]["first_observed_attainment"]) self.assertEqual(group["runs"][0]["full_declared_adaptation"]["cost_by_currency"], {"USD": "2.8"}) def test_missing_assessment_makes_nonattainment_indeterminate(self): self.manifest["quality_threshold"] = .75 self.quality.fixtures[2].set_values(self.quality.fixtures[2].forms[0]["ratings"][0], [None]+[1]*5+[0]*6) report = self.analyze() group = self.group(report) self.assertEqual(group["quality_outcome_counts"]["indeterminate"], 1) self.assertEqual(group["threshold_analysis_outcome_counts"]["fully_observed_non_attainment"], 23) self.assertIsNone(group["observed_attainment_rate"]) self.assertIsNone(group["quality_attainment_rate"]) def test_observed_hit_survives_missing_earlier_assessment_with_firstness_limit(self): self.quality.fixtures[1].set_values(self.quality.fixtures[1].forms[0]["ratings"][0], [None]+[0]*11) row = self.target(self.analyze()) self.assertEqual(row["quality_outcome"], "observed_attainment") self.assertEqual(row["first_observed_attainment"]["turn"], 2) self.assertFalse(row["first_observed_attainment"]["no_missing_assessments_before_hit"]) self.assertFalse(row["trajectory_complete"]) def test_unknown_billing_is_not_zero_and_does_not_erase_quality_witness(self): target = self.orders[self.rid][2] next(r for r in self.ledger["requests"] if r["request_id"] == target)["billed_cost"]["amount"] = None group = self.group(self.analyze()) row = next(r for r in group["runs"] if r["run_id"] == self.rid) self.assertEqual(row["quality_outcome"], "observed_attainment") self.assertEqual(row["threshold_analysis_outcome"], "indeterminate") hit = row["first_observed_attainment"] self.assertIsNone(hit["actual_billed_cost_by_currency"]) self.assertEqual(hit["cumulative_adaptation"]["resources"]["billing"]["by_currency"]["USD"]["known_subtotal"], "1.2") self.assertIsNone(group["observed_attainment_rate"]) def test_missing_tail_cost_does_not_change_known_first_hit_cost(self): target = self.orders[self.rid][-1] next(r for r in self.ledger["requests"] if r["request_id"] == target)["billed_cost"]["amount"] = None row = self.target(self.analyze()) self.assertEqual(row["first_observed_attainment"]["actual_billed_cost_by_currency"], {"USD": "1.5"}) self.assertEqual(row["threshold_analysis_outcome"], "observed_attainment") self.assertFalse(row["trajectory_complete"]) def test_missing_first_hit_boundary_never_substitutes_later_costed_hit(self): self.quality.fixtures[3].set_values(self.quality.fixtures[3].forms[0]["ratings"][0], [1]*12) self.manifest["run_accounting"][0]["boundaries"][1].update(cumulative_request_count=None, unavailability_reason="Artificial missing boundary") row = self.target(self.analyze()) self.assertEqual(row["first_observed_attainment"]["turn"], 2) self.assertIsNone(row["first_observed_attainment"]["actual_billed_cost_by_currency"]) self.assertEqual(row["threshold_analysis_outcome"], "indeterminate") def test_mixed_currencies_remain_a_vector_and_unknown_currency_is_missing(self): target = self.orders[self.rid][1] request = next(r for r in self.ledger["requests"] if r["request_id"] == target) request["billed_cost"]["currency"] = "EUR" row = self.target(self.analyze()) self.assertEqual(row["first_observed_attainment"]["actual_billed_cost_by_currency"], {"EUR": "0.2", "USD": "1.3"}) self.assertIsNone(row["first_observed_attainment"]["cumulative_adaptation"]["resources"]["billing"]["single_currency_billed_total"]) request["billed_cost"]["currency"] = None self.assertIsNone(self.target(self.analyze())["first_observed_attainment"]["actual_billed_cost_by_currency"]) def test_partial_ledger_records_and_incomplete_log_withhold_totals(self): omitted = self.orders[self.rid][1] self.ledger["requests"] = [r for r in self.ledger["requests"] if r["request_id"] != omitted] self.ledger["runs"][0]["phase_log_complete"]["adaptation"] = False row = self.target(self.analyze()) prefix = row["first_observed_attainment"]["cumulative_adaptation"] self.assertEqual(prefix["missing_request_ids"], [omitted]) self.assertFalse(prefix["coverage_complete"]) self.assertIsNone(prefix["cost_by_currency"]) def test_unavailable_ledger_and_missing_expected_run_are_not_zero_cost(self): rid = self.rid self.ledger["runs"] = [r for r in self.ledger["runs"] if r["run_id"] != rid] self.ledger["requests"] = [r for r in self.ledger["requests"] if r["run_id"] != rid] self.manifest["run_accounting"][0].update(ledger_id=None, unavailability_reason="Artificial unavailable billing") row = self.target(self.analyze()) self.assertIsNone(row["first_observed_attainment"]["actual_billed_cost_by_currency"]) self.assertEqual(row["threshold_analysis_outcome"], "indeterminate") self.quality.manifest["runs"] = [r for r in self.quality.manifest["runs"] if r["run_id"] != rid] self.manifest["run_accounting"] = [r for r in self.manifest["run_accounting"] if r["run_id"] != rid] group = self.group(self.analyze()) self.assertEqual((group["expected_run_denominator"], group["observed_run_count"]), (24, 23)) self.assertEqual(group["incomplete_trajectory_count"], 1) self.assertIsNone(group["observed_attainment_rate"]) def test_assessment_charge_cannot_masquerade_as_an_adaptation_request(self): self.ledger["runs"][0]["phase_log_complete"]["adaptation"] = False request = next(r for r in self.ledger["requests"] if r["request_id"] == self.orders[self.rid][0]) request.update(phase="final_assessment", role="scorer") self.prepare() with self.assertRaisesRegex(threshold.sealing.ValidationError, "attempt identity mismatch"): threshold.analyze_threshold(self.manifest_path, self.quality.archive.registry_path) def test_threshold_validation_and_exact_comparison(self): for bad in (-.1, 1.1, True, "0.5", None): self.manifest["quality_threshold"] = bad self.prepare() with self.assertRaises((threshold.sealing.ValidationError, threshold.accounting.LedgerError)): threshold.analyze_threshold(self.manifest_path, self.quality.archive.registry_path) self.assertTrue(threshold.meets_threshold(1/12, Decimal("0.083333333333333333"))) self.assertFalse(threshold.meets_threshold(1/12, Decimal("0.083333333333333334"))) self.manifest["quality_threshold"] = 0 self.assertEqual(self.group(self.analyze())["runs"][0]["first_observed_attainment"]["turn"], 1) def test_invalid_boundaries_order_identity_and_missing_attempts_rejected(self): original = copy.deepcopy(self.manifest["run_accounting"][0]) cases = ( (lambda r: r["adaptation_request_order"].pop(), "every sealed adaptation attempt"), (lambda r: r["adaptation_request_order"].reverse(), "request chronology"), (lambda r: r["boundaries"][1].update(cumulative_request_count=100), "out-of-range"), (lambda r: r["boundaries"][0].update(cumulative_request_count=5), "after snapshot"), (lambda r: r["boundaries"][1].update(cumulative_request_count=1), "earlier submission"), (lambda r: r["boundaries"][0].update(artifact_sha256="0"*64), "artifact identity/hash mismatch"), (lambda r: r.update(condition_id="wrong-condition"), "run identity or version"), (lambda r: r["boundaries"].append(copy.deepcopy(r["boundaries"][0])), "duplicate/reversed boundary"), ) for change, error in cases: self.manifest["run_accounting"][0] = copy.deepcopy(original) change(self.manifest["run_accounting"][0]) self.prepare() with self.assertRaisesRegex(threshold.sealing.ValidationError, error): threshold.analyze_threshold(self.manifest_path, self.quality.archive.registry_path) def test_request_identity_duplicate_and_contradictory_complete_log_rejected(self): original = copy.deepcopy(self.ledger) for change, error in ( (lambda l: l["requests"].append(copy.deepcopy(l["requests"][0])), "duplicate request"), (lambda l: l["runs"][0].update(task_id="wrong-task"), "ledger run identity mismatch"), (lambda l: l["requests"].pop(1), "complete adaptation ledger omits"), (lambda l: l["requests"][1].update(role="other"), "attempt identity mismatch"), (lambda l: l.update(ledger_version="2.0"), "unsupported ledger_version"), ): self.ledger = copy.deepcopy(original) change(self.ledger) self.prepare() with self.assertRaisesRegex((threshold.sealing.ValidationError, threshold.accounting.LedgerError), error): threshold.analyze_threshold(self.manifest_path, self.quality.archive.registry_path) def test_hashes_and_unsealed_inputs_rejected(self): self.prepare() self.manifest["ledgers"][0]["sha256"] = "0"*64 self.save_manifest() with self.assertRaisesRegex(threshold.sealing.ValidationError, "source hash mismatch"): threshold.analyze_threshold(self.manifest_path, self.quality.archive.registry_path) self.prepare() (self.quality.archive.output/"packet-build-manifest.json").unlink() with self.assertRaisesRegex(threshold.sealing.ValidationError, "unsealed"): threshold.analyze_threshold(self.manifest_path, self.quality.archive.registry_path) def test_output_no_overwrites_and_read_only_inputs(self): self.prepare() inputs = [self.manifest_path, self.quality.manifest_path, self.ledger_path] inputs += [p for p in self.quality.archive.output.rglob("*") if p.is_file()] inputs += [p for fixture in self.quality.fixtures.values() for p in fixture.directory.glob("*.json")] before = {str(p): p.read_bytes() for p in inputs} threshold.write_report(self.manifest_path, self.output, self.quality.archive.registry_path) self.assertEqual(json.loads(self.output.read_bytes())["threshold_report_version"], "1.0") self.assertEqual(before, {str(p): p.read_bytes() for p in inputs}) with self.assertRaisesRegex(threshold.sealing.ValidationError, "existing output"): threshold.write_report(self.manifest_path, self.output, self.quality.archive.registry_path) with self.assertRaisesRegex(threshold.sealing.ValidationError, "outside sealed archives"): threshold.write_report(self.manifest_path, self.quality.archive.output/"forbidden.json", self.quality.archive.registry_path) def test_failed_publication_never_emits_success_or_report(self): self.prepare() stdout, stderr = io.StringIO(), io.StringIO() with mock.patch.object(threshold.os, "link", side_effect=OSError("artificial publication failure")): with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = threshold.main(["--manifest", str(self.manifest_path), "--registry", str(self.quality.archive.registry_path), "--output", str(self.output)]) self.assertEqual(code, 2) self.assertFalse(self.output.exists()) self.assertEqual(stdout.getvalue(), "") if __name__ == "__main__": unittest.main()