"""Temporary artificial metadata fixtures; no study outcomes or task answers.""" import copy import contextlib import io import json import unittest import seal_runs as sealing import summarize_run_outcomes as outcomes import test_seal_runs as fixtures class OutcomeTests(unittest.TestCase): def fixture(self): fixture = fixtures.SealingTests() fixture.setUp() self.addCleanup(fixture.doCleanups) return fixture def remove_usage(self, fixture): fixture.source["records"] = [r for r in fixture.source["records"] if r["record_id"] != "fixture-usage"] for run in fixture.source["runs"]: run["record_ids"] = [r for r in run["record_ids"] if r != "fixture-usage"] fixture.set_availability() for run in fixture.source["runs"]: run["record_availability"]["usage_records"] = dict(state="unavailable", note="Artificial fixture: no usage available") def report(self, fixture, *others): return outcomes.summarize_archives([f.output for f in (fixture,)+others], fixture.registry_path) def mutate_inventory(self, fixture, change): path = fixture.output/"archive-inventory.json" value = json.loads(path.read_bytes()) change(value) data = sealing.json_bytes(value) path.write_bytes(data) (fixture.output/"archive-inventory.sha256").write_text(sealing.sha(data)+"\n") def test_counts_denominators_and_no_completion_inference(self): f = self.fixture() f.seal() report = self.report(f) total = report["overall"] self.assertEqual(total["expected_run_cells"], 12) self.assertEqual(total["observed_run_denominator"], 2) self.assertEqual(total["absent_run_cell_count"], 10) self.assertEqual(total["status_counts"], dict(completed=1, incomplete=1, failed=0, cancelled=0)) self.assertEqual(total["status_rates_among_observed"]["completed"], dict(numerator=1, denominator=2, rate=.5)) self.assertEqual(total["endpoint_availability"]["available"], 2) self.assertFalse(report["requirement_satisfaction_evaluated"]) empty = next(v for v in report["by_family"].values() if v["observed_run_denominator"] == 0) self.assertEqual(empty["absent_run_cell_count"], 4) self.assertIsNone(empty["status_rates_among_observed"]["failed"]["rate"]) def test_four_statuses_and_unassessable_run_retained(self): f = self.fixture() for index, status in ((4, "failed"), (8, "cancelled")): run = copy.deepcopy(f.source["runs"][1]) run.update(run_id="fixture-"+status, task_id=f.registry["tasks"][index]["task_id"], run_status=status) run["record_ids"] = ["fixture-config", "fixture-prompt-b"] run["attempts"][0].update(attempt_id="attempt-"+status, status=status) run["endpoint_selection"] = dict(artifact_id=None, basis="Explicit artificial missing endpoint") if status == "cancelled": f.add_record("cancelled-endpoint", "artifact", b"Artificial text fixture only.\n") run["record_ids"].append("cancelled-endpoint") run["endpoint_selection"] = dict(artifact_id="cancelled-endpoint", basis="Explicit artificial endpoint") f.source["runs"].append(run) self.remove_usage(f) f.seal() report = self.report(f) self.assertEqual(report["overall"]["status_counts"], {s: 1 for s in sealing.RUN_STATUSES}) self.assertEqual(report["overall"]["endpoint_availability"], dict(available=3, not_designated=1, denominator=4)) self.assertEqual(report["overall"]["absent_run_cell_count"], 8) self.assertEqual(sum(len(v["absent_cells"]) for v in report["by_family"].values()), 8) packet = json.loads((f.output/"packet-build-manifest.json").read_bytes()) self.assertEqual(len(packet["runs"]), 3) self.assertEqual(len(report["runs"]), 4) condition = report["by_condition"]["fixture-condition"] self.assertEqual(condition["summary"], report["overall"]) for family, summary in report["by_family"].items(): self.assertEqual(condition["by_family"][family], summary) def second_shard(self, first, new_repetition=False, same_cells=False): second = self.fixture() self.remove_usage(second) if new_repetition: second.source["conditions"][0]["repetition_ids"] = ["fixture-repetition-two"] for index, run in enumerate(second.source["runs"]): run["run_id"] = "shard-two-"+str(index) if not same_cells: run["task_id"] = second.registry["tasks"][index+2]["task_id"] if new_repetition: run["repetition_id"] = "fixture-repetition-two" first.seal() second.seal() return second def test_union_declarations_do_not_multiply_expected_cells(self): first = self.fixture() second = self.second_shard(first) report = self.report(first, second) self.assertEqual(report["overall"]["expected_run_cells"], 12) self.assertEqual(report["overall"]["observed_run_denominator"], 4) self.assertEqual(report["overall"]["absent_run_cell_count"], 8) def test_distinct_declared_repetitions_expand_expected_cells(self): first = self.fixture() second = self.second_shard(first, new_repetition=True) report = self.report(first, second) self.assertEqual(report["overall"]["expected_run_cells"], 24) self.assertEqual(report["overall"]["absent_run_cell_count"], 20) def test_wholly_absent_condition_not_classified_as_failure(self): f = self.fixture() f.source["conditions"].append(dict(condition_id="absent-condition", repetition_ids=["rep"] )) f.seal() report = self.report(f) group = report["by_condition"]["absent-condition"]["summary"] self.assertEqual(group["expected_run_cells"], 12) self.assertEqual(group["observed_run_denominator"], 0) self.assertEqual(group["status_counts"], {s: 0 for s in sealing.RUN_STATUSES}) self.assertTrue(all(x["rate"] is None for x in group["status_rates_among_observed"].values())) def test_duplicate_archive_and_cross_archive_cell_rejected(self): first = self.fixture() second = self.second_shard(first, same_cells=True) with self.assertRaisesRegex(sealing.ValidationError, "duplicate archive"): self.report(first, first) with self.assertRaisesRegex(sealing.ValidationError, "duplicate run identity or run cell"): self.report(first, second) def test_unsealed_missing_or_false_commit_rejected(self): for missing in (True, False): with self.subTest(missing=missing): f = self.fixture() f.seal() path = f.output/"packet-build-manifest.json" if missing: path.unlink() else: packet = json.loads(path.read_bytes()) packet["batch_sealed"] = False path.write_bytes(sealing.json_bytes(packet)) with self.assertRaisesRegex(sealing.ValidationError, "unsealed"): self.report(f) def test_incompatible_versions_rejected(self): for target in ("archive", "commit", "registry"): with self.subTest(target=target): f = self.fixture() f.seal() if target == "archive": self.mutate_inventory(f, lambda value: value.update(archive_inventory_version="2.0")) elif target == "commit": path = f.output/"packet-build-manifest.json" packet = json.loads(path.read_bytes()) packet["build_manifest_version"] = "2.0" path.write_bytes(sealing.json_bytes(packet)) else: f.registry["tasks"][0]["task_version"] = "2.0" f.registry_path.write_bytes(sealing.json_bytes(f.registry)) with self.assertRaisesRegex(sealing.ValidationError, "incompatible"): self.report(f) def test_tampered_payload_and_inconsistent_commit_rejected(self): f = self.fixture() f.seal() inventory = json.loads((f.output/"archive-inventory.json").read_bytes()) path = f.output/inventory["records"][0]["path"] original = path.read_bytes() path.write_bytes(original+b"tamper") with self.assertRaisesRegex(sealing.ValidationError, "copy validation"): self.report(f) path.write_bytes(original) packet_path = f.output/"packet-build-manifest.json" packet = json.loads(packet_path.read_bytes()) packet["runs"].pop() packet_path.write_bytes(sealing.json_bytes(packet)) with self.assertRaisesRegex(sealing.ValidationError, "inconsistent"): self.report(f) def test_original_source_paths_never_dereferenced(self): f = self.fixture() f.seal() for record in f.source["records"]: (f.root/record["path"]).unlink() f.input_path.unlink() f.registry_path.unlink() report = outcomes.summarize_archives([f.output], f.output/"registry/task-registry.json") self.assertEqual(report["overall"]["observed_run_denominator"], 2) def test_symlink_escape_rejected_before_payload_read(self): f = self.fixture() f.seal() inventory = json.loads((f.output/"archive-inventory.json").read_bytes()) record = inventory["records"][0] path = f.output/record["path"] path.unlink() path.symlink_to(f.root/(record["record_id"]+".source")) with self.assertRaisesRegex(sealing.ValidationError, "outside archive"): self.report(f) def test_cli_read_only_and_invalid_input_has_no_report(self): f = self.fixture() f.seal() before = {str(p.relative_to(f.output)): p.read_bytes() for p in f.output.rglob("*") if p.is_file()} stdout, stderr = io.StringIO(), io.StringIO() args = ["--archives", str(f.output), "--registry", str(f.registry_path)] with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): self.assertEqual(outcomes.main(args), 0) self.assertEqual(json.loads(stdout.getvalue())["outcome_report_version"], "1.0") self.assertEqual(before, {str(p.relative_to(f.output)): p.read_bytes() for p in f.output.rglob("*") if p.is_file()}) (f.output/"packet-build-manifest.json").unlink() stdout = io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): self.assertEqual(outcomes.main(args), 2) self.assertEqual(stdout.getvalue(), "") if __name__ == "__main__": unittest.main()