"""Temporary artificial B3 parser/archive/numeric fixtures, never study findings.""" import copy import html import json import unittest from unittest import mock import audit_boundary as audit import b3_contract as b3 import test_initial_contract as fixtures class B3AuditTests(unittest.TestCase): def setUp(self): self.base = fixtures.InitialAuditTests() self.base.setUp() self.addCleanup(self.base.doCleanups) self.f, self.sealer = self.base.f, self.base.sealer self.run = self.sealer.source["runs"][1] self.run["condition_id"] = "B3" self.sealer.source["conditions"][1]["condition_id"] = "B3" ledger = json.loads(self.sealer.contents["fixture-usage"]) ledger["runs"][1]["condition_id"] = "B3" self.replace("fixture-usage", audit.sealing.json_bytes(ledger)) self.f.manifest["audit_manifest_version"] = "1.2" self.task = self.base.task_packet self.tid = self.run["task_id"] self.p0 = b3.initial_contract.TEMPLATE_PATH.read_bytes().decode().split("IMMUTABLE_TASK_PACKET_JSON\n")[0].rstrip("\r\n") self.initial_id = "fixture-request-b" initial_attempt = next(a for a in self.run["attempts"] if a["attempt_id"] == self.initial_id) # Replace the fixture's old continuation attempt explicitly, not by parsing text. old = next(a for a in self.run["attempts"] if a["attempt_id"] != self.initial_id) self.run["attempts"].remove(old) if "fixture-b-endpoint" not in initial_attempt["record_ids"]: initial_attempt["record_ids"].append("fixture-b-endpoint") source = self.f.manifest["runs"][1] self.row = dict(**{k: source[k] for k in (*audit.IDENTITY, "archive_id", "seal_record_id")}, audit_scope="b3-native", contract=b3.CONTRACT, packet_inventory_complete=True, states=[], initialization=dict(state_id="s0", worker_attempts=[dict(attempt_id=self.initial_id, artifact_record_id="fixture-b-endpoint", artifact_sha256=audit.sealing.sha(self.sealer.contents["fixture-b-endpoint"]))], artifact_attempt_id=self.initial_id), iterations=[], final_incumbent_id="s0", packets=[]) self.f.manifest["runs"][1] = self.row self.add_state("s0", self.p0, "fixture-b-endpoint") initial_raw = self.render("worker", dict(CANDIDATE_PROMPT_JSON=self.p0, TASK_PACKET_JSON=self.task)) self.replace("fixture-prompt-b", initial_raw) self.row["packets"].append(self.packet("init-packet", self.initial_id, 1, "fixture-prompt-b", initial_raw)) self.add_state("s1", "Artificial candidate directive & parser fixture; no task answer.", "b3-artifact-1", b"Artificial candidate text. Caf\xc3\xa9.\r\n") self.build_iteration("it1", "s0", "s1", 2) self.reseal() def replace(self, rid, raw): (self.sealer.root/(rid+".source")).write_bytes(raw) self.sealer.contents[rid] = raw self.f.packet_bytes[rid] = raw def add_record(self, rid, kind, raw): self.sealer.add_record(rid, kind, raw) self.run["record_ids"].append(rid) self.f.packet_bytes[rid] = raw def add_state(self, sid, prompt, rid, raw=None): if raw is not None: self.add_record(rid, "artifact", raw) self.row["states"].append(dict(state_id=sid, prompt=prompt, artifact_record_id=rid, artifact_sha256=audit.sealing.sha(self.sealer.contents[rid]))) def state(self, sid): return next(s for s in self.row["states"] if s["state_id"] == sid) def render(self, stage, values, pretty=False): raw = (b3.BASE/b3.TEMPLATES[stage][0]).read_bytes().decode() for name, value in values.items(): raw = raw.replace("{{"+name+"}}", json.dumps(value, ensure_ascii=False, indent=2 if pretty else None)) return raw.encode() def packet(self, pid, aid, seq, rid, raw): return dict(input_scope="native", packet_id=pid, attempt_id=aid, sequence=seq, record_id=rid, sha256=audit.sealing.sha(raw)) def add_call(self, aid, role, stage, seq, values, response=None, artifact=None): rid = aid+"-input" raw = self.render(stage, values) self.add_record(rid, "submitted_prompt", raw) refs = [rid] self.row["packets"].append(self.packet(aid+"-packet", aid, seq, rid, raw)) if artifact: refs.append(artifact) descriptor = dict(attempt_id=aid, artifact_record_id=artifact, artifact_sha256=audit.sealing.sha(self.sealer.contents[artifact])) else: response_id = aid+"-response" self.add_record(response_id, "operational_record", response) refs.append(response_id) descriptor = dict(attempt_id=aid, response_state="available", response_record_id=response_id, response_sha256=audit.sealing.sha(response)) self.run["attempts"].append(dict(attempt_id=aid, phase="adaptation", role=role, status="succeeded", retry_of=None, started_at_utc=None, ended_at_utc=None, record_ids=refs)) return descriptor def decision(self, inc, cand, inc_votes=1, cand_votes=3, complete=True): return dict(incumbent_id=inc, candidate_id=cand, comparison_complete=complete, incumbent_votes=inc_votes if complete else None, candidate_votes=cand_votes if complete else None, accept_candidate=complete and cand_votes > inc_votes, selected_id=cand if complete and cand_votes > inc_votes else inc, reason="incomplete_keep_incumbent" if not complete else "candidate_majority" if cand_votes > inc_votes else "tie_keep_incumbent" if cand_votes == inc_votes else "incumbent_majority") def build_iteration(self, iid, inc, cand, sequence, tied=False): incumbent, candidate = self.state(inc), self.state(cand) gen_id, worker_id = iid+"-generation", iid+"-worker" gen = self.add_call(gen_id, "optimizer", "generation", sequence, dict(TASK_PACKET_JSON=self.task, INCUMBENT_PROMPT_JSON=incumbent["prompt"], INCUMBENT_ARTIFACT_JSON=dict(artifact_id=incumbent["artifact_record_id"], text=self.sealer.contents[incumbent["artifact_record_id"]].decode())), ("Artificial parser fixture.Artificial change."+ html.escape(candidate["prompt"])+"").encode()) worker = self.add_call(worker_id, "worker", "worker", sequence+1, dict(TASK_PACKET_JSON=self.task, CANDIDATE_PROMPT_JSON=candidate["prompt"]), artifact=candidate["artifact_record_id"]) rounds, records = [], [] for n in range(1, 5): a, b = (inc, cand) if n % 2 else (cand, inc) winner = cand if n <= (2 if tied else 3) else inc choice = "A" if winner == a else "B" values = dict(PUBLIC_TASKS_JSON=[dict(task_id=self.tid, task_packet=self.task)], OUTPUTS_A_JSON=[dict(task_id=self.tid, text=self.sealer.contents[self.state(a)["artifact_record_id"]].decode())], OUTPUTS_B_JSON=[dict(task_id=self.tid, text=self.sealer.contents[self.state(b)["artifact_record_id"]].decode())]) desc = self.add_call(iid+"-comparison-"+str(n), "comparator", "comparison", sequence+1+n, values, ("Artificial numeric fixture."+choice+"").encode()) rounds.append(dict(round=n, a_id=a, b_id=b, attempts=[desc], vote_attempt_id=desc["attempt_id"])) records.append(dict(round=n, a_id=a, b_id=b, choice=choice)) result = self.decision(inc, cand, 2 if tied else 1, 2 if tied else 3) selection_id = iid+"-selection-record" self.add_record(selection_id, "operational_record", audit.sealing.json_bytes(dict(selection_record_version="1.0", records=records, result=result))) iteration = dict(iteration_id=iid, sequence=len(self.row["iterations"])+1, incumbent_id=inc, candidate_id=cand, generation_attempts=[gen], generation_attempt_id=gen_id, worker_attempts=[worker], artifact_attempt_id=worker_id, comparisons=rounds, selection=dict(state="available", record_id=selection_id, sha256=audit.sealing.sha(self.sealer.contents[selection_id]), selector_attempts=[], result_attempt_id=None), next_incumbent_id=result["selected_id"]) self.row["iterations"].append(iteration) self.row["final_incumbent_id"] = result["selected_id"] self.run["endpoint_selection"]["artifact_id"] = self.state(result["selected_id"])["artifact_record_id"] return iteration @property def iteration(self): return self.row["iterations"][0] def reseal(self): self.base.base.reseal() def mutate_input(self, aid, stage, values): p = next(p for p in self.row["packets"] if p["attempt_id"] == aid) raw = self.render(stage, values) self.replace(p["record_id"], raw) p["sha256"] = audit.sealing.sha(raw) def update_selection(self, envelope): sel = self.iteration["selection"] raw = audit.sealing.json_bytes(envelope) self.replace(sel["record_id"], raw) sel["sha256"] = audit.sealing.sha(raw) def envelope(self): return json.loads(self.sealer.contents[self.iteration["selection"]["record_id"]]) def report(self, value=0): return self.base.result(self.base.completed(value)) def native(self, report): return report["series"]["independent:artificial-reviewer-a"]["by_condition"]["B3"] def test_valid_native_sequence_preserves_bytes_and_distinct_initialization(self): context = self.f.prepare() self.assertEqual(context["audit_context_version"], "1.2") kinds = {c["kind"] for c in context["copies"]} self.assertTrue({"b3_initialization_input", "b3_generation_input", "b3_worker_input", "b3_comparison_input"} <= kinds) self.assertEqual(len([c for c in context["copies"] if c["kind"].startswith("b3_")]), 7) for c in context["copies"]: if c["kind"].startswith("b3_"): packet = next(p for p in self.row["packets"] if p["packet_id"] == c["identity"]) self.assertEqual((self.f.output/c["path"]).read_bytes(), self.sealer.contents[packet["record_id"]]) report = self.report() self.assertEqual(report["audit_report_version"], "1.2") stats = self.native(report) self.assertEqual(stats["counts"]["compliant"], 1) self.assertEqual(stats["b3_native_coverage"]["incomplete_iteration_count"], 0) self.assertEqual(stats["input_scope_coverage"]["initial"]["out_of_scope_run_count"], 1) self.assertFalse(report["whole_run_compliance_certified"]) self.assertFalse(report["semantic_absence_of_restricted_information_certified"]) def test_tie_keeps_incumbent_and_rejected_candidate_is_retained(self): envelope = self.envelope() record = envelope["records"][2] record["choice"] = "A" # Round 3 A is incumbent: yields 2--2. aid = self.iteration["comparisons"][2]["vote_attempt_id"] descriptor = self.iteration["comparisons"][2]["attempts"][0] raw = b"Artificial tie fixture.A" self.replace(descriptor["response_record_id"], raw) descriptor["response_sha256"] = audit.sealing.sha(raw) envelope["result"] = self.decision("s0", "s1", 2, 2) self.update_selection(envelope) self.iteration["next_incumbent_id"] = self.row["final_incumbent_id"] = "s0" self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"] self.reseal() self.f.prepare() stats = self.native(self.report())["b3_native_coverage"] self.assertEqual(stats["rejected_candidate_count"], 1) self.assertIn("b3-artifact-1", self.run["record_ids"]) def test_promotion_and_next_generation_follow_recorded_incumbent(self): self.add_state("s2", "Artificial second changed directive; no task answer.", "b3-artifact-2", b"Artificial later text.\r\n") self.build_iteration("it2", "s1", "s2", 8, tied=True) self.reseal() self.f.prepare() stats = self.native(self.report())["b3_native_coverage"] self.assertEqual(stats["iteration_denominator"], 2) self.assertEqual(stats["rejected_candidate_count"], 1) def test_wrong_next_incumbent_and_wrong_final_endpoint_rejected(self): self.iteration["next_incumbent_id"] = "s0" with self.assertRaisesRegex(ValueError, "selection/next incumbent"): self.f.prepare() self.iteration["next_incumbent_id"] = "s1" self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"] self.reseal() with self.assertRaisesRegex(ValueError, "prescribed endpoint"): self.f.prepare() def test_altered_wrappers_wrong_hashes_and_no_history_projection(self): for stage in b3.TEMPLATES: raw = self.render(stage, {n: "Artificial parser fixture" for n in b3.TEMPLATES[stage][2]}) with self.assertRaises(ValueError): b3.decode_input(raw+b"extra", stage, self.f.manifest["bounds"]) with self.assertRaises(ValueError): b3.decode_input(b"extra\n"+raw, stage, self.f.manifest["bounds"]) with mock.patch.dict(b3.TEMPLATES, {"worker": (b3.TEMPLATES["worker"][0], "0"*64, b3.TEMPLATES["worker"][2])}): with self.assertRaisesRegex(ValueError, "template hash"): b3.decode_input(b"fixture", "worker", self.f.manifest["bounds"]) self.mutate_input("it1-worker", "worker", dict(CANDIDATE_PROMPT_JSON=self.state("s1")["prompt"], TASK_PACKET_JSON=self.task+"changed")) self.reseal() with self.assertRaisesRegex(ValueError, "public packet hash"): self.f.prepare() def test_worker_prompt_must_be_exact_xml_decoded_generation(self): self.state("s1")["prompt"] = "Invented different directive." with self.assertRaisesRegex(ValueError, "XML-decoded candidate prompt"): self.f.prepare() def test_anonymous_bundles_mapping_and_hidden_fields_rejected(self): aid = "it1-comparison-1" p = next(p for p in self.row["packets"] if p["attempt_id"] == aid) values = b3.decode_input(self.sealer.contents[p["record_id"]], "comparison", self.f.manifest["bounds"]) values["OUTPUTS_A_JSON"][0]["prompt_id"] = "s0" self.mutate_input(aid, "comparison", values) self.reseal() with self.assertRaises(ValueError): self.f.prepare() values["OUTPUTS_A_JSON"][0].pop("prompt_id") values["OUTPUTS_A_JSON"][0]["text"] = values["OUTPUTS_B_JSON"][0]["text"] self.mutate_input(aid, "comparison", values) self.reseal() with self.assertRaisesRegex(ValueError, "anonymous A/B mapping"): self.f.prepare() def test_wrong_selection_counts_and_recorded_ab_mapping_rejected(self): env = self.envelope() env["result"]["candidate_votes"] = 4 self.update_selection(env) self.reseal() with self.assertRaisesRegex(ValueError, "four-vote rule"): self.f.prepare() env["result"] = self.decision("s0", "s1", 1, 3) env["records"][0]["a_id"], env["records"][0]["b_id"] = "s1", "s0" # Preserve a 3--1 decision while reversing the source mapping. env["records"][0]["choice"] = "A" self.update_selection(env) self.reseal() with self.assertRaisesRegex(ValueError, "declared A/B mapping"): self.f.prepare() def test_missing_request_and_response_evidence_remains_unknown(self): self.row["packets"][1].update(record_id=None, sha256=None) desc = self.iteration["comparisons"][0]["attempts"][0] desc.update(response_state="unavailable", response_record_id=None, response_sha256=None) self.f.prepare() stats = self.native(self.report()) self.assertEqual(stats["counts"]["unassessed"], 1) self.assertEqual(stats["b3_native_coverage"]["unavailable_response_count"], 1) self.assertEqual(stats["b3_native_coverage"]["missing_input_count"], 1) def test_missing_comparison_round_forces_recorded_incomplete_keep(self): rnd = self.iteration["comparisons"][3] aid = rnd["vote_attempt_id"] self.run["attempts"] = [a for a in self.run["attempts"] if a["attempt_id"] != aid] self.row["packets"] = [p for p in self.row["packets"] if p["attempt_id"] != aid] rnd.update(attempts=[], vote_attempt_id=None) env = self.envelope() env["records"][3]["choice"] = None env["result"] = self.decision("s0", "s1", complete=False) self.update_selection(env) self.iteration["next_incumbent_id"] = self.row["final_incumbent_id"] = "s0" self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"] self.reseal() self.f.prepare() stats = self.native(self.report())["b3_native_coverage"] self.assertEqual(stats["incomplete_iteration_count"], 1) native_row = self.report()["series"]["independent:artificial-reviewer-a"]["runs"][1] self.assertEqual(native_row["b3_native_coverage"]["iterations"][0]["not_attempted_comparison_count"], 1) def test_selection_record_unavailable_never_certifies_incumbent_chain(self): self.iteration["selection"].update(state="unavailable", record_id=None, sha256=None) self.f.prepare() self.assertEqual(self.native(self.report())["counts"]["unassessed"], 1) def test_failure_and_retry_attempts_are_not_filtered(self): rnd = self.iteration["comparisons"][0] original = next(a for a in self.run["attempts"] if a["attempt_id"] == rnd["vote_attempt_id"]) retry = dict(original, attempt_id="b3-comparison-retry", retry_of=original["attempt_id"], status="failed") self.run["attempts"].append(retry) rnd["attempts"].append(dict(rnd["attempts"][0], attempt_id=retry["attempt_id"])) last = self.row["packets"][-1] source = next(p for p in self.row["packets"] if p["attempt_id"] == original["attempt_id"]) self.row["packets"].append(dict(source, packet_id="retry-packet", attempt_id=retry["attempt_id"], sequence=last["sequence"]+1)) self.reseal() self.f.prepare() stats = self.native(self.report())["b3_native_coverage"] self.assertEqual(stats["retry_attempt_count"], 1) native_row = self.report()["series"]["independent:artificial-reviewer-a"]["runs"][1] self.assertEqual(native_row["b3_native_coverage"]["attempt_status_counts"]["failed"], 1) def test_duplicate_stages_missing_slots_and_wrong_roles_rejected(self): old = copy.deepcopy(self.row) for case in ("duplicate", "slot", "mapping", "contract"): with self.subTest(case=case): if case == "duplicate": self.iteration["generation_attempts"].append(copy.deepcopy(self.iteration["generation_attempts"][0])) elif case == "slot": self.row["packets"].pop(1) elif case == "mapping": self.iteration["comparisons"][0]["a_id"] = "unseen" else: self.row["contract"] = "b3-spo-native/2.0" with self.assertRaises(ValueError): self.f.prepare() self.row.clear() self.row.update(copy.deepcopy(old)) def test_semantic_human_judgments_and_other_series_stay_separate(self): self.f.prepare() report = self.report(1) self.assertEqual(self.native(report)["counts"]["noncompliant"], 1) self.assertEqual(report["series"]["adjudicated"]["overall"]["counts"]["unassessed"], 2) self.assertEqual(report["series"]["independent:artificial-reviewer-a"]["overall"]["assigned_run_denominator"], 2) def test_no_overwrites_unsealed_archives_and_scope_version_rejected(self): original = self.f.manifest["audit_manifest_version"] self.f.manifest["audit_manifest_version"] = "1.1" with self.assertRaises(ValueError): self.f.prepare() self.f.manifest["audit_manifest_version"] = original self.f.prepare() with self.assertRaises(ValueError): self.f.prepare() (self.sealer.output/"packet-build-manifest.json").unlink() with self.assertRaisesRegex(ValueError, "unsealed"): self.report() def test_optional_model_selector_request_and_output_are_bound_to_record(self): env = self.envelope() descriptor = self.add_call("it1-selector", "selector", "selection", 8, dict(INCUMBENT_ID_JSON="s0", CANDIDATE_ID_JSON="s1", COMPARISON_RECORDS_JSON=env["records"]), audit.sealing.json_bytes(env["result"])) self.iteration["selection"].update(selector_attempts=[descriptor], result_attempt_id="it1-selector") self.reseal() self.f.prepare() stats = self.native(self.report()) self.assertEqual(stats["counts"]["compliant"], 1) native_row = self.report()["series"]["independent:artificial-reviewer-a"]["runs"][1] self.assertEqual(native_row["b3_native_coverage"]["stage_attempt_counts"]["selection"], 1) def test_malformed_comparison_is_invalid_not_a_synthesized_vote(self): descriptor = self.iteration["comparisons"][0]["attempts"][0] raw = b"Artificial invalid fixture.tie" self.replace(descriptor["response_record_id"], raw) descriptor["response_sha256"] = audit.sealing.sha(raw) env = self.envelope() env["records"][0]["choice"] = None env["result"] = self.decision("s0", "s1", complete=False) self.update_selection(env) self.iteration["next_incumbent_id"] = self.row["final_incumbent_id"] = "s0" self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"] self.reseal() self.f.prepare() stats = self.native(self.report())["b3_native_coverage"] self.assertEqual(stats["incomplete_iteration_count"], 1) self.assertEqual(stats["incomplete_selection_retention_count"], 1) self.assertEqual(stats["rejected_candidate_count"], 0) def test_interrupted_invalid_proposal_retains_old_incumbent_and_failure(self): descriptor = self.iteration["generation_attempts"][0] raw = b"Artificial malformed XML fixture, not a proposal." self.replace(descriptor["response_record_id"], raw) descriptor["response_sha256"] = audit.sealing.sha(raw) keep = {self.initial_id, descriptor["attempt_id"]} self.run["attempts"] = [a for a in self.run["attempts"] if a["attempt_id"] in keep] next(a for a in self.run["attempts"] if a["attempt_id"] == descriptor["attempt_id"])["status"] = "failed" self.row["packets"] = [p for p in self.row["packets"] if p["attempt_id"] in keep] self.iteration.update(generation_attempt_id=None, worker_attempts=[], artifact_attempt_id=None, next_incumbent_id="s0") for rnd in self.iteration["comparisons"]: rnd.update(attempts=[], vote_attempt_id=None) self.iteration["selection"].update(state="not_executed", record_id=None, sha256=None) self.state("s1").update(prompt=None, artifact_record_id=None, artifact_sha256=None) self.row["final_incumbent_id"] = "s0" self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"] self.reseal() self.f.prepare() stats = self.native(self.report())["b3_native_coverage"] self.assertEqual(stats["incomplete_iteration_count"], 1) native_row = self.report()["series"]["independent:artificial-reviewer-a"]["runs"][1] self.assertEqual(native_row["b3_native_coverage"]["attempt_status_counts"]["failed"], 1) self.assertEqual(native_row["b3_native_coverage"]["iterations"][0]["generation_invalid_response_count"], 1) def test_invalid_record_or_missing_vote_must_keep_incumbent_with_null_counts(self): env = self.envelope() env["records"] = env["records"][:3] result = self.decision("s0", "s1", complete=False) self.assertFalse(b3.selection_consistent(env["records"], result, "s0", "s1")) result["candidate_votes"] = 3 with self.assertRaisesRegex(ValueError, "four-vote rule"): b3.selection_consistent(env["records"], result, "s0", "s1") result = self.decision("s0", "s1", complete=False) duplicated = [env["records"][0]]*4 self.assertFalse(b3.selection_consistent(duplicated, result, "s0", "s1")) def test_packet_hash_response_provenance_and_duplicate_json_are_checked(self): original = self.row["packets"][1]["sha256"] self.row["packets"][1]["sha256"] = "0"*64 with self.assertRaisesRegex(ValueError, "hash mismatch"): self.f.prepare() self.row["packets"][1]["sha256"] = original desc = self.iteration["generation_attempts"][0] desc["response_sha256"] = "0"*64 with self.assertRaisesRegex(ValueError, "hash mismatch"): self.f.prepare() raw = self.render("worker", dict(CANDIDATE_PROMPT_JSON="fixture", TASK_PACKET_JSON=self.task)) raw = raw.replace(b'"fixture"', b'{"id":1,"id":2}') with self.assertRaises(ValueError): b3.decode_input(raw, "worker", self.f.manifest["bounds"]) def test_xml_escaping_does_not_forward_analysis_and_typed_selection_is_strict(self): parsed = b3.xml_response(b"fixturefixtureA & B", "generation") self.assertEqual(parsed["prompt"], "A & B") self.assertIsNone(b3.xml_response(b"x A ", "comparison")) env = self.envelope() env["result"]["comparison_complete"] = 1 with self.assertRaises(ValueError): b3.selection_consistent(env["records"], env["result"], "s0", "s1") def test_independent_and_adjudicated_forms_remain_separate(self): self.f.prepare() forms = [self.base.completed(0), self.base.completed(1, "independent-2.blank.json"), self.base.completed(0, "adjudicated.blank.json")] for row in forms[-1]["ratings"]: row.update(independent_judgments=[0, 1], resolution_note="Artificial resolution fixture; no study judgment.") paths = [self.f.write_form(form, str(i)+".json") for i, form in enumerate(forms)] report = self.f.summarize(paths) self.assertEqual(self.native(report)["counts"]["compliant"], 1) self.assertEqual(report["series"]["independent:artificial-reviewer-b"]["by_condition"]["B3"]["counts"]["noncompliant"], 1) self.assertEqual(report["series"]["adjudicated"]["by_condition"]["B3"]["counts"]["compliant"], 1) def test_missing_declared_candidate_state_is_unknown_not_reconstructed(self): self.state("s1").update(prompt=None, artifact_record_id=None, artifact_sha256=None) self.f.prepare() self.assertEqual(self.native(self.report())["counts"]["unassessed"], 1) def test_explicitly_unproduced_generation_cannot_feed_worker(self): self.iteration["generation_attempts"][0].update(response_state="not_produced", response_record_id=None, response_sha256=None) with self.assertRaisesRegex(ValueError, "unproduced generation"): self.f.prepare() def test_missing_next_and_final_links_are_unknown_not_inferred(self): self.iteration["next_incumbent_id"] = self.row["final_incumbent_id"] = None self.f.prepare() self.assertEqual(self.native(self.report())["counts"]["unassessed"], 1) def test_missing_archived_endpoint_keeps_failed_run_in_denominator(self): self.run["run_status"] = "failed" self.run["endpoint_selection"]["artifact_id"] = None self.reseal() self.f.prepare() report = self.report() self.assertEqual(self.native(report)["counts"]["unassessed"], 1) self.assertEqual(report["series"]["independent:artificial-reviewer-a"]["overall"]["assigned_run_denominator"], 2) if __name__ == "__main__": unittest.main()