"""Temporary artificial text, assessment and accounting fixtures; no study findings.""" import contextlib from decimal import Decimal import io import json from pathlib import Path import sys import unittest from unittest import mock import analyze_equal_cost as equal sys.path.insert(0, str(Path(__file__).resolve().parent.parent/"cost-to-quality")) import test_analyze_cost_threshold as fixtures class EqualCostTests(unittest.TestCase): def setUp(self): self.source = fixtures.ThresholdTests() self.source.setUp() self.addCleanup(self.source.doCleanups) self.root = self.source.quality.archive.root self.manifest_path = self.root/"equal-cost-selection.json" self.output = self.root/"new-equal-cost-report.json" self.manifest = dict(equal_cost_manifest_version="1.0", analysis_id="artificial-equal-cost-analysis", common_caps=[dict(currency="USD", amounts=[.5, 1.5, 3])], progress_manifest={}, ledgers=[], run_accounting=self.source.manifest["run_accounting"]) def prepare(self): self.source.prepare() self.manifest["progress_manifest"] = self.source.manifest["progress_manifest"] self.manifest["ledgers"] = self.source.manifest["ledgers"] self.save() def save(self): self.manifest_path.write_bytes(equal.sealing.json_bytes(self.manifest)) def analyze(self): self.prepare() return equal.analyze_equal_cost(self.manifest_path, self.source.quality.archive.registry_path) def panel(self, report, cap=3, currency="USD", rater="analysis-rater-a"): panels = report["independent"][rater]["by_currency"][currency]["caps"] return next(p for p in panels if Decimal(p["cap_amount"]) == Decimal(str(cap))) def group(self, report, cap=3, currency="USD", rater="analysis-rater-a"): return self.panel(report, cap, currency, rater)["by_condition"][self.source.quality.condition] def target(self, report, cap=3, currency="USD", rater="analysis-rater-a"): return next(r for r in self.group(report, cap, currency, rater)["runs"] if r["run_id"] == self.source.rid) def test_latest_affordable_snapshot_not_highest_quality_and_slack_is_separate(self): report = self.analyze() row = self.target(report) chosen = row["selected_snapshot"] self.assertEqual((chosen["turn"], chosen["observed_quality"]), (3, .25)) self.assertEqual((chosen["actual_spend"], chosen["unused_cap"]), ("2.8", "0.2")) earlier = self.target(report, 1.5)["selected_snapshot"] self.assertEqual((earlier["turn"], earlier["observed_quality"]), (2, .5)) self.assertEqual(Decimal(earlier["unused_cap"]), 0) self.assertEqual(row["prescribed_endpoint"]["listed_turn"], 3) self.assertFalse(row["prescribed_endpoint"]["endpoint_changed"]) self.assertEqual(self.target(report, .5)["selected_snapshot"]["turn"], 1) self.assertEqual(self.target(report, .5)["prescribed_endpoint"]["listed_turn"], 3) def test_failed_rejected_discarded_retry_costs_count_and_final_assessment_is_separate(self): self.manifest["common_caps"][0]["amounts"] = [2.5] row = self.target(self.analyze(), 2.5) self.assertEqual(row["selected_snapshot"]["turn"], 2) resources = row["selected_snapshot"]["cumulative_adaptation"]["resources"] self.assertEqual(resources["recorded_request_count"], 5) self.assertEqual(resources["retry_attempt_count"], 1) self.assertEqual(resources["status_counts"]["failed"], 1) self.assertEqual(resources["disposition_counts"]["rejected"], 2) self.assertEqual(row["final_assessment_resources_separate"]["billing"]["single_currency_billed_total"]["amount"], "99") def test_equal_spend_ties_choose_later_turn(self): for request in self.source.ledger["requests"]: if request["request_id"] in self.source.orders[self.source.rid][-2:]: request["billed_cost"]["amount"] = 0 chosen = self.target(self.analyze(), 1.5)["selected_snapshot"] self.assertEqual(chosen["turn"], 3) self.assertEqual(chosen["observed_quality"], .25) self.assertEqual(Decimal(chosen["actual_spend"]), Decimal("1.5")) def test_selected_missing_rating_never_falls_back_and_series_stay_separate(self): fixture = self.source.quality.fixtures[3] fixture.set_values(fixture.forms[0]["ratings"][0], [None]+[1]*11) report = self.analyze() row = self.target(report) self.assertEqual(row["selection_state"], "selected") self.assertEqual(row["selected_snapshot"]["turn"], 3) self.assertIsNone(row["selected_snapshot"]["observed_quality"]) self.assertEqual(row["comparison_state"], "missing_selected_assessment") self.assertIsNone(self.group(report)["quality_summary"]["mean"]) self.assertFalse(self.panel(report)["complete_cohort_comparison"]) self.assertEqual(self.target(report, rater="analysis-rater-b")["selected_snapshot"]["turn"], 3) self.assertEqual(self.group(report, rater="analysis-rater-b")["quality_summary"]["mean"], 1) self.assertEqual(report["adjudicated"]["by_currency"]["USD"]["caps"][-1]["by_condition"][self.source.quality.condition]["quality_summary"]["mean"], 1) def test_missing_nonselected_rating_does_not_block_observed_quality_at_cap(self): fixture = self.source.quality.fixtures[1] fixture.set_values(fixture.forms[0]["ratings"][0], [None]+[0]*11) report = self.analyze() self.assertEqual(self.group(report)["quality_summary"]["mean"], .25) self.assertTrue(self.group(report)["quality_coverage_complete"]) def test_later_unknown_billing_can_block_selection_or_be_proved_over_cap(self): request = next(r for r in self.source.ledger["requests"] if r["request_id"] == self.source.orders[self.source.rid][-1]) request["billed_cost"]["amount"] = None report = self.analyze() row = self.target(report) self.assertEqual(row["selection_state"], "indeterminate") self.assertIsNone(row["selected_snapshot"]) self.assertEqual(row["latest_known_eligible_candidate_not_selected"], 2) self.assertEqual(row["blocking_turns"], [3]) self.assertIsNone(self.group(report)["quality_summary"]["mean"]) lower_cap = self.target(report, 1.5) self.assertEqual(lower_cap["selected_snapshot"]["turn"], 2) self.assertFalse(lower_cap["all_snapshot_accounting_known"]) self.assertEqual(lower_cap["eligibility_audit"][-1]["state"], "over_cap") def test_missing_earlier_boundary_does_not_outrank_a_later_known_candidate(self): self.manifest["run_accounting"][0]["boundaries"][1].update(cumulative_request_count=None, unavailability_reason="Artificial missing middle boundary") report = self.analyze() self.assertEqual(self.target(report)["selected_snapshot"]["turn"], 3) self.assertEqual(self.target(report, 1.5)["selection_state"], "indeterminate") def test_known_over_cap_barrier_handles_a_later_unavailable_snapshot(self): self.source.quality.manifest["runs"][0]["snapshots"].pop() self.manifest["run_accounting"][0]["boundaries"].pop() report = self.analyze() self.assertEqual(self.target(report)["selection_state"], "indeterminate") low = self.target(report, .5) self.assertEqual(low["selected_snapshot"]["turn"], 1) self.assertEqual(low["eligibility_audit"][-1]["reason"], "after_established_over_cap_prefix") def test_currency_vectors_are_not_ignored_or_converted(self): request = next(r for r in self.source.ledger["requests"] if r["request_id"] == self.source.orders[self.source.rid][1]) request["billed_cost"]["currency"] = "EUR" self.manifest["common_caps"].append(dict(currency="EUR", amounts=[3])) report = self.analyze() row = self.target(report) self.assertEqual(row["selection_state"], "indeterminate") self.assertIn("incomparable_currency_vector", row["indeterminacy_reasons"]) self.assertIsNone(row["selected_snapshot"]) self.assertEqual(self.target(report, currency="EUR")["selection_state"], "indeterminate") self.assertFalse(self.panel(report, currency="EUR")["complete_cohort_comparison"]) def test_no_affordable_snapshot_is_not_zero_quality(self): self.manifest["common_caps"][0]["amounts"] = [0] report = self.analyze() group = self.group(report, 0) self.assertEqual(group["comparison_state_counts"]["no_snapshot_within_cap"], 24) self.assertTrue(group["selection_coverage_complete"]) self.assertIsNone(group["quality_summary"]["mean"]) self.assertFalse(group["quality_coverage_complete"]) self.assertIsNone(self.target(report, 0)["selected_snapshot"]) def test_zero_observed_quality_is_not_missing(self): report = self.analyze() self.assertEqual(self.group(report, .5)["quality_summary"]["mean"], 0) self.assertTrue(self.group(report, .5)["quality_coverage_complete"]) self.assertEqual(self.target(report, .5)["comparison_state"], "observed_quality") def test_common_condition_comparison_withheld_when_one_cohort_missing(self): self.source.quality.manifest["conditions"].append(dict(condition_id="absent-fixture-condition", repetition_ids=[self.source.quality.repetitions[0]])) report = self.analyze() self.assertEqual(self.group(report)["quality_summary"]["mean"], .25) panel = self.panel(report) self.assertFalse(panel["complete_cohort_comparison"]) self.assertIsNone(panel["complete_condition_quality_means"]) self.assertEqual(panel["by_condition"]["absent-fixture-condition"]["expected_run_denominator"], 12) def test_equal_task_and_family_weighting_and_no_missing_run_renormalization(self): fixture = self.source.quality.fixtures[3] family = self.source.quality.archive.registry["families"][0] tasks = family["task_ids"] for assignment, row in zip(fixture.manifest["assignments"], fixture.forms[0]["ratings"]): count = 12 if assignment["task_id"] not in tasks or (assignment["task_id"] == tasks[0] and assignment["repetition_id"] == self.source.quality.repetitions[0]) else 0 fixture.set_values(row, [1]*count+[0]*(12-count)) report = self.analyze() aggregate = self.group(report)["quality_summary"] self.assertEqual(aggregate["task_means"][tasks[0]], .5) self.assertEqual(aggregate["family_means"][family["family_id"]], .125) self.assertAlmostEqual(aggregate["mean"], (.125+1+1)/3) self.source.quality.manifest["runs"].pop(0) self.manifest["run_accounting"].pop(0) report = self.analyze() self.assertEqual(self.group(report)["expected_run_denominator"], 24) self.assertEqual(self.group(report)["observed_run_count"], 23) self.assertIsNone(self.group(report)["quality_summary"]["mean"]) def test_exact_decimal_slack(self): self.manifest["common_caps"][0]["amounts"] = [10**30] report = self.analyze() chosen = self.target(report, 10**30)["selected_snapshot"] self.assertEqual(chosen["unused_cap"], str(10**30-3)+".2") def test_invalid_caps_duplicates_currency_and_versions_rejected(self): bad_panels = [[], [dict(currency="usd", amounts=[1])], [dict(currency="USD", amounts=[])], [dict(currency="USD", amounts=[True])], [dict(currency="USD", amounts=[-1])], [dict(currency="USD", amounts=["1"])], [dict(currency="USD", amounts=[None])], [dict(currency="USD", amounts=[2, 1])], [dict(currency="USD", amounts=[1, 1.0])], [dict(currency="USD", amounts=[1]), dict(currency="USD", amounts=[2])]] self.prepare() for caps in bad_panels: self.manifest["common_caps"] = caps self.save() with self.assertRaises((equal.sealing.ValidationError, equal.accounting.LedgerError)): equal.analyze_equal_cost(self.manifest_path, self.source.quality.archive.registry_path) self.manifest["common_caps"] = [dict(currency="USD", amounts=[1])] self.manifest["equal_cost_manifest_version"] = "2.0" self.save() with self.assertRaisesRegex(equal.sealing.ValidationError, "incompatible"): equal.analyze_equal_cost(self.manifest_path, self.source.quality.archive.registry_path) def test_hash_chronology_boundary_and_unsealed_rejections(self): self.prepare() self.manifest["ledgers"][0]["sha256"] = "0"*64 self.save() with self.assertRaisesRegex(equal.sealing.ValidationError, "source hash mismatch"): equal.analyze_equal_cost(self.manifest_path, self.source.quality.archive.registry_path) self.prepare() self.manifest["run_accounting"][0]["boundaries"][0]["cumulative_request_count"] = 5 self.save() with self.assertRaisesRegex(equal.sealing.ValidationError, "after snapshot"): equal.analyze_equal_cost(self.manifest_path, self.source.quality.archive.registry_path) self.manifest["run_accounting"][0]["boundaries"][0]["cumulative_request_count"] = 1 self.prepare() (self.source.quality.archive.output/"packet-build-manifest.json").unlink() with self.assertRaisesRegex(equal.sealing.ValidationError, "unsealed"): equal.analyze_equal_cost(self.manifest_path, self.source.quality.archive.registry_path) def test_read_only_inputs_and_no_overwrites(self): self.prepare() inputs = [self.manifest_path, self.source.quality.manifest_path, self.source.ledger_path] inputs += [p for p in self.source.quality.archive.output.rglob("*") if p.is_file()] inputs += [p for fixture in self.source.quality.fixtures.values() for p in fixture.directory.glob("*.json")] before = {str(p): p.read_bytes() for p in inputs} equal.write_report(self.manifest_path, self.output, self.source.quality.archive.registry_path) self.assertTrue(json.loads(self.output.read_bytes())["coordinator_only"]) self.assertEqual(before, {str(p): p.read_bytes() for p in inputs}) with self.assertRaisesRegex(equal.sealing.ValidationError, "existing output"): equal.write_report(self.manifest_path, self.output, self.source.quality.archive.registry_path) with self.assertRaisesRegex(equal.sealing.ValidationError, "outside sealed archives"): equal.write_report(self.manifest_path, self.source.quality.archive.output/"forbidden.json", self.source.quality.archive.registry_path) def test_failed_publication_emits_no_success_or_report(self): self.prepare() stdout, stderr = io.StringIO(), io.StringIO() with mock.patch.object(equal.os, "link", side_effect=OSError("artificial publication failure")): with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = equal.main(["--manifest", str(self.manifest_path), "--registry", str(self.source.quality.archive.registry_path), "--output", str(self.output)]) self.assertEqual(code, 2) self.assertFalse(self.output.exists()) self.assertEqual(stdout.getvalue(), "") if __name__ == "__main__": unittest.main()