"""Temporary artificial B3 parser/archive/numeric fixtures, never study findings."""
import copy
import html
import json
import unittest
from unittest import mock
import audit_boundary as audit
import b3_contract as b3
import test_initial_contract as fixtures
class B3AuditTests(unittest.TestCase):
def setUp(self):
self.base = fixtures.InitialAuditTests()
self.base.setUp()
self.addCleanup(self.base.doCleanups)
self.f, self.sealer = self.base.f, self.base.sealer
self.run = self.sealer.source["runs"][1]
self.run["condition_id"] = "B3"
self.sealer.source["conditions"][1]["condition_id"] = "B3"
ledger = json.loads(self.sealer.contents["fixture-usage"])
ledger["runs"][1]["condition_id"] = "B3"
self.replace("fixture-usage", audit.sealing.json_bytes(ledger))
self.f.manifest["audit_manifest_version"] = "1.2"
self.task = self.base.task_packet
self.tid = self.run["task_id"]
self.p0 = b3.initial_contract.TEMPLATE_PATH.read_bytes().decode().split("IMMUTABLE_TASK_PACKET_JSON\n")[0].rstrip("\r\n")
self.initial_id = "fixture-request-b"
initial_attempt = next(a for a in self.run["attempts"] if a["attempt_id"] == self.initial_id)
# Replace the fixture's old continuation attempt explicitly, not by parsing text.
old = next(a for a in self.run["attempts"] if a["attempt_id"] != self.initial_id)
self.run["attempts"].remove(old)
if "fixture-b-endpoint" not in initial_attempt["record_ids"]:
initial_attempt["record_ids"].append("fixture-b-endpoint")
source = self.f.manifest["runs"][1]
self.row = dict(**{k: source[k] for k in (*audit.IDENTITY, "archive_id", "seal_record_id")}, audit_scope="b3-native",
contract=b3.CONTRACT, packet_inventory_complete=True, states=[],
initialization=dict(state_id="s0", worker_attempts=[dict(attempt_id=self.initial_id,
artifact_record_id="fixture-b-endpoint", artifact_sha256=audit.sealing.sha(self.sealer.contents["fixture-b-endpoint"]))],
artifact_attempt_id=self.initial_id), iterations=[], final_incumbent_id="s0", packets=[])
self.f.manifest["runs"][1] = self.row
self.add_state("s0", self.p0, "fixture-b-endpoint")
initial_raw = self.render("worker", dict(CANDIDATE_PROMPT_JSON=self.p0, TASK_PACKET_JSON=self.task))
self.replace("fixture-prompt-b", initial_raw)
self.row["packets"].append(self.packet("init-packet", self.initial_id, 1, "fixture-prompt-b", initial_raw))
self.add_state("s1", "Artificial candidate directive & parser fixture; no task answer.", "b3-artifact-1", b"Artificial candidate text. Caf\xc3\xa9.\r\n")
self.build_iteration("it1", "s0", "s1", 2)
self.reseal()
def replace(self, rid, raw):
(self.sealer.root/(rid+".source")).write_bytes(raw)
self.sealer.contents[rid] = raw
self.f.packet_bytes[rid] = raw
def add_record(self, rid, kind, raw):
self.sealer.add_record(rid, kind, raw)
self.run["record_ids"].append(rid)
self.f.packet_bytes[rid] = raw
def add_state(self, sid, prompt, rid, raw=None):
if raw is not None:
self.add_record(rid, "artifact", raw)
self.row["states"].append(dict(state_id=sid, prompt=prompt, artifact_record_id=rid,
artifact_sha256=audit.sealing.sha(self.sealer.contents[rid])))
def state(self, sid):
return next(s for s in self.row["states"] if s["state_id"] == sid)
def render(self, stage, values, pretty=False):
raw = (b3.BASE/b3.TEMPLATES[stage][0]).read_bytes().decode()
for name, value in values.items():
raw = raw.replace("{{"+name+"}}", json.dumps(value, ensure_ascii=False, indent=2 if pretty else None))
return raw.encode()
def packet(self, pid, aid, seq, rid, raw):
return dict(input_scope="native", packet_id=pid, attempt_id=aid, sequence=seq, record_id=rid, sha256=audit.sealing.sha(raw))
def add_call(self, aid, role, stage, seq, values, response=None, artifact=None):
rid = aid+"-input"
raw = self.render(stage, values)
self.add_record(rid, "submitted_prompt", raw)
refs = [rid]
self.row["packets"].append(self.packet(aid+"-packet", aid, seq, rid, raw))
if artifact:
refs.append(artifact)
descriptor = dict(attempt_id=aid, artifact_record_id=artifact, artifact_sha256=audit.sealing.sha(self.sealer.contents[artifact]))
else:
response_id = aid+"-response"
self.add_record(response_id, "operational_record", response)
refs.append(response_id)
descriptor = dict(attempt_id=aid, response_state="available", response_record_id=response_id, response_sha256=audit.sealing.sha(response))
self.run["attempts"].append(dict(attempt_id=aid, phase="adaptation", role=role, status="succeeded", retry_of=None,
started_at_utc=None, ended_at_utc=None, record_ids=refs))
return descriptor
def decision(self, inc, cand, inc_votes=1, cand_votes=3, complete=True):
return dict(incumbent_id=inc, candidate_id=cand, comparison_complete=complete,
incumbent_votes=inc_votes if complete else None, candidate_votes=cand_votes if complete else None,
accept_candidate=complete and cand_votes > inc_votes,
selected_id=cand if complete and cand_votes > inc_votes else inc,
reason="incomplete_keep_incumbent" if not complete else "candidate_majority" if cand_votes > inc_votes else
"tie_keep_incumbent" if cand_votes == inc_votes else "incumbent_majority")
def build_iteration(self, iid, inc, cand, sequence, tied=False):
incumbent, candidate = self.state(inc), self.state(cand)
gen_id, worker_id = iid+"-generation", iid+"-worker"
gen = self.add_call(gen_id, "optimizer", "generation", sequence,
dict(TASK_PACKET_JSON=self.task, INCUMBENT_PROMPT_JSON=incumbent["prompt"],
INCUMBENT_ARTIFACT_JSON=dict(artifact_id=incumbent["artifact_record_id"], text=self.sealer.contents[incumbent["artifact_record_id"]].decode())),
("Artificial parser fixture.Artificial change."+
html.escape(candidate["prompt"])+"").encode())
worker = self.add_call(worker_id, "worker", "worker", sequence+1,
dict(TASK_PACKET_JSON=self.task, CANDIDATE_PROMPT_JSON=candidate["prompt"]), artifact=candidate["artifact_record_id"])
rounds, records = [], []
for n in range(1, 5):
a, b = (inc, cand) if n % 2 else (cand, inc)
winner = cand if n <= (2 if tied else 3) else inc
choice = "A" if winner == a else "B"
values = dict(PUBLIC_TASKS_JSON=[dict(task_id=self.tid, task_packet=self.task)],
OUTPUTS_A_JSON=[dict(task_id=self.tid, text=self.sealer.contents[self.state(a)["artifact_record_id"]].decode())],
OUTPUTS_B_JSON=[dict(task_id=self.tid, text=self.sealer.contents[self.state(b)["artifact_record_id"]].decode())])
desc = self.add_call(iid+"-comparison-"+str(n), "comparator", "comparison", sequence+1+n, values,
("Artificial numeric fixture."+choice+"").encode())
rounds.append(dict(round=n, a_id=a, b_id=b, attempts=[desc], vote_attempt_id=desc["attempt_id"]))
records.append(dict(round=n, a_id=a, b_id=b, choice=choice))
result = self.decision(inc, cand, 2 if tied else 1, 2 if tied else 3)
selection_id = iid+"-selection-record"
self.add_record(selection_id, "operational_record", audit.sealing.json_bytes(dict(selection_record_version="1.0", records=records, result=result)))
iteration = dict(iteration_id=iid, sequence=len(self.row["iterations"])+1, incumbent_id=inc, candidate_id=cand,
generation_attempts=[gen], generation_attempt_id=gen_id, worker_attempts=[worker], artifact_attempt_id=worker_id,
comparisons=rounds, selection=dict(state="available", record_id=selection_id, sha256=audit.sealing.sha(self.sealer.contents[selection_id]),
selector_attempts=[], result_attempt_id=None), next_incumbent_id=result["selected_id"])
self.row["iterations"].append(iteration)
self.row["final_incumbent_id"] = result["selected_id"]
self.run["endpoint_selection"]["artifact_id"] = self.state(result["selected_id"])["artifact_record_id"]
return iteration
@property
def iteration(self):
return self.row["iterations"][0]
def reseal(self):
self.base.base.reseal()
def mutate_input(self, aid, stage, values):
p = next(p for p in self.row["packets"] if p["attempt_id"] == aid)
raw = self.render(stage, values)
self.replace(p["record_id"], raw)
p["sha256"] = audit.sealing.sha(raw)
def update_selection(self, envelope):
sel = self.iteration["selection"]
raw = audit.sealing.json_bytes(envelope)
self.replace(sel["record_id"], raw)
sel["sha256"] = audit.sealing.sha(raw)
def envelope(self):
return json.loads(self.sealer.contents[self.iteration["selection"]["record_id"]])
def report(self, value=0):
return self.base.result(self.base.completed(value))
def native(self, report):
return report["series"]["independent:artificial-reviewer-a"]["by_condition"]["B3"]
def test_valid_native_sequence_preserves_bytes_and_distinct_initialization(self):
context = self.f.prepare()
self.assertEqual(context["audit_context_version"], "1.2")
kinds = {c["kind"] for c in context["copies"]}
self.assertTrue({"b3_initialization_input", "b3_generation_input", "b3_worker_input", "b3_comparison_input"} <= kinds)
self.assertEqual(len([c for c in context["copies"] if c["kind"].startswith("b3_")]), 7)
for c in context["copies"]:
if c["kind"].startswith("b3_"):
packet = next(p for p in self.row["packets"] if p["packet_id"] == c["identity"])
self.assertEqual((self.f.output/c["path"]).read_bytes(), self.sealer.contents[packet["record_id"]])
report = self.report()
self.assertEqual(report["audit_report_version"], "1.2")
stats = self.native(report)
self.assertEqual(stats["counts"]["compliant"], 1)
self.assertEqual(stats["b3_native_coverage"]["incomplete_iteration_count"], 0)
self.assertEqual(stats["input_scope_coverage"]["initial"]["out_of_scope_run_count"], 1)
self.assertFalse(report["whole_run_compliance_certified"])
self.assertFalse(report["semantic_absence_of_restricted_information_certified"])
def test_tie_keeps_incumbent_and_rejected_candidate_is_retained(self):
envelope = self.envelope()
record = envelope["records"][2]
record["choice"] = "A" # Round 3 A is incumbent: yields 2--2.
aid = self.iteration["comparisons"][2]["vote_attempt_id"]
descriptor = self.iteration["comparisons"][2]["attempts"][0]
raw = b"Artificial tie fixture.A"
self.replace(descriptor["response_record_id"], raw)
descriptor["response_sha256"] = audit.sealing.sha(raw)
envelope["result"] = self.decision("s0", "s1", 2, 2)
self.update_selection(envelope)
self.iteration["next_incumbent_id"] = self.row["final_incumbent_id"] = "s0"
self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"]
self.reseal()
self.f.prepare()
stats = self.native(self.report())["b3_native_coverage"]
self.assertEqual(stats["rejected_candidate_count"], 1)
self.assertIn("b3-artifact-1", self.run["record_ids"])
def test_promotion_and_next_generation_follow_recorded_incumbent(self):
self.add_state("s2", "Artificial second changed directive; no task answer.", "b3-artifact-2", b"Artificial later text.\r\n")
self.build_iteration("it2", "s1", "s2", 8, tied=True)
self.reseal()
self.f.prepare()
stats = self.native(self.report())["b3_native_coverage"]
self.assertEqual(stats["iteration_denominator"], 2)
self.assertEqual(stats["rejected_candidate_count"], 1)
def test_wrong_next_incumbent_and_wrong_final_endpoint_rejected(self):
self.iteration["next_incumbent_id"] = "s0"
with self.assertRaisesRegex(ValueError, "selection/next incumbent"):
self.f.prepare()
self.iteration["next_incumbent_id"] = "s1"
self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"]
self.reseal()
with self.assertRaisesRegex(ValueError, "prescribed endpoint"):
self.f.prepare()
def test_altered_wrappers_wrong_hashes_and_no_history_projection(self):
for stage in b3.TEMPLATES:
raw = self.render(stage, {n: "Artificial parser fixture" for n in b3.TEMPLATES[stage][2]})
with self.assertRaises(ValueError):
b3.decode_input(raw+b"extra", stage, self.f.manifest["bounds"])
with self.assertRaises(ValueError):
b3.decode_input(b"extra\n"+raw, stage, self.f.manifest["bounds"])
with mock.patch.dict(b3.TEMPLATES, {"worker": (b3.TEMPLATES["worker"][0], "0"*64, b3.TEMPLATES["worker"][2])}):
with self.assertRaisesRegex(ValueError, "template hash"):
b3.decode_input(b"fixture", "worker", self.f.manifest["bounds"])
self.mutate_input("it1-worker", "worker", dict(CANDIDATE_PROMPT_JSON=self.state("s1")["prompt"], TASK_PACKET_JSON=self.task+"changed"))
self.reseal()
with self.assertRaisesRegex(ValueError, "public packet hash"):
self.f.prepare()
def test_worker_prompt_must_be_exact_xml_decoded_generation(self):
self.state("s1")["prompt"] = "Invented different directive."
with self.assertRaisesRegex(ValueError, "XML-decoded candidate prompt"):
self.f.prepare()
def test_anonymous_bundles_mapping_and_hidden_fields_rejected(self):
aid = "it1-comparison-1"
p = next(p for p in self.row["packets"] if p["attempt_id"] == aid)
values = b3.decode_input(self.sealer.contents[p["record_id"]], "comparison", self.f.manifest["bounds"])
values["OUTPUTS_A_JSON"][0]["prompt_id"] = "s0"
self.mutate_input(aid, "comparison", values)
self.reseal()
with self.assertRaises(ValueError):
self.f.prepare()
values["OUTPUTS_A_JSON"][0].pop("prompt_id")
values["OUTPUTS_A_JSON"][0]["text"] = values["OUTPUTS_B_JSON"][0]["text"]
self.mutate_input(aid, "comparison", values)
self.reseal()
with self.assertRaisesRegex(ValueError, "anonymous A/B mapping"):
self.f.prepare()
def test_wrong_selection_counts_and_recorded_ab_mapping_rejected(self):
env = self.envelope()
env["result"]["candidate_votes"] = 4
self.update_selection(env)
self.reseal()
with self.assertRaisesRegex(ValueError, "four-vote rule"):
self.f.prepare()
env["result"] = self.decision("s0", "s1", 1, 3)
env["records"][0]["a_id"], env["records"][0]["b_id"] = "s1", "s0"
# Preserve a 3--1 decision while reversing the source mapping.
env["records"][0]["choice"] = "A"
self.update_selection(env)
self.reseal()
with self.assertRaisesRegex(ValueError, "declared A/B mapping"):
self.f.prepare()
def test_missing_request_and_response_evidence_remains_unknown(self):
self.row["packets"][1].update(record_id=None, sha256=None)
desc = self.iteration["comparisons"][0]["attempts"][0]
desc.update(response_state="unavailable", response_record_id=None, response_sha256=None)
self.f.prepare()
stats = self.native(self.report())
self.assertEqual(stats["counts"]["unassessed"], 1)
self.assertEqual(stats["b3_native_coverage"]["unavailable_response_count"], 1)
self.assertEqual(stats["b3_native_coverage"]["missing_input_count"], 1)
def test_missing_comparison_round_forces_recorded_incomplete_keep(self):
rnd = self.iteration["comparisons"][3]
aid = rnd["vote_attempt_id"]
self.run["attempts"] = [a for a in self.run["attempts"] if a["attempt_id"] != aid]
self.row["packets"] = [p for p in self.row["packets"] if p["attempt_id"] != aid]
rnd.update(attempts=[], vote_attempt_id=None)
env = self.envelope()
env["records"][3]["choice"] = None
env["result"] = self.decision("s0", "s1", complete=False)
self.update_selection(env)
self.iteration["next_incumbent_id"] = self.row["final_incumbent_id"] = "s0"
self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"]
self.reseal()
self.f.prepare()
stats = self.native(self.report())["b3_native_coverage"]
self.assertEqual(stats["incomplete_iteration_count"], 1)
native_row = self.report()["series"]["independent:artificial-reviewer-a"]["runs"][1]
self.assertEqual(native_row["b3_native_coverage"]["iterations"][0]["not_attempted_comparison_count"], 1)
def test_selection_record_unavailable_never_certifies_incumbent_chain(self):
self.iteration["selection"].update(state="unavailable", record_id=None, sha256=None)
self.f.prepare()
self.assertEqual(self.native(self.report())["counts"]["unassessed"], 1)
def test_failure_and_retry_attempts_are_not_filtered(self):
rnd = self.iteration["comparisons"][0]
original = next(a for a in self.run["attempts"] if a["attempt_id"] == rnd["vote_attempt_id"])
retry = dict(original, attempt_id="b3-comparison-retry", retry_of=original["attempt_id"], status="failed")
self.run["attempts"].append(retry)
rnd["attempts"].append(dict(rnd["attempts"][0], attempt_id=retry["attempt_id"]))
last = self.row["packets"][-1]
source = next(p for p in self.row["packets"] if p["attempt_id"] == original["attempt_id"])
self.row["packets"].append(dict(source, packet_id="retry-packet", attempt_id=retry["attempt_id"], sequence=last["sequence"]+1))
self.reseal()
self.f.prepare()
stats = self.native(self.report())["b3_native_coverage"]
self.assertEqual(stats["retry_attempt_count"], 1)
native_row = self.report()["series"]["independent:artificial-reviewer-a"]["runs"][1]
self.assertEqual(native_row["b3_native_coverage"]["attempt_status_counts"]["failed"], 1)
def test_duplicate_stages_missing_slots_and_wrong_roles_rejected(self):
old = copy.deepcopy(self.row)
for case in ("duplicate", "slot", "mapping", "contract"):
with self.subTest(case=case):
if case == "duplicate":
self.iteration["generation_attempts"].append(copy.deepcopy(self.iteration["generation_attempts"][0]))
elif case == "slot":
self.row["packets"].pop(1)
elif case == "mapping":
self.iteration["comparisons"][0]["a_id"] = "unseen"
else:
self.row["contract"] = "b3-spo-native/2.0"
with self.assertRaises(ValueError):
self.f.prepare()
self.row.clear()
self.row.update(copy.deepcopy(old))
def test_semantic_human_judgments_and_other_series_stay_separate(self):
self.f.prepare()
report = self.report(1)
self.assertEqual(self.native(report)["counts"]["noncompliant"], 1)
self.assertEqual(report["series"]["adjudicated"]["overall"]["counts"]["unassessed"], 2)
self.assertEqual(report["series"]["independent:artificial-reviewer-a"]["overall"]["assigned_run_denominator"], 2)
def test_no_overwrites_unsealed_archives_and_scope_version_rejected(self):
original = self.f.manifest["audit_manifest_version"]
self.f.manifest["audit_manifest_version"] = "1.1"
with self.assertRaises(ValueError):
self.f.prepare()
self.f.manifest["audit_manifest_version"] = original
self.f.prepare()
with self.assertRaises(ValueError):
self.f.prepare()
(self.sealer.output/"packet-build-manifest.json").unlink()
with self.assertRaisesRegex(ValueError, "unsealed"):
self.report()
def test_optional_model_selector_request_and_output_are_bound_to_record(self):
env = self.envelope()
descriptor = self.add_call("it1-selector", "selector", "selection", 8,
dict(INCUMBENT_ID_JSON="s0", CANDIDATE_ID_JSON="s1", COMPARISON_RECORDS_JSON=env["records"]),
audit.sealing.json_bytes(env["result"]))
self.iteration["selection"].update(selector_attempts=[descriptor], result_attempt_id="it1-selector")
self.reseal()
self.f.prepare()
stats = self.native(self.report())
self.assertEqual(stats["counts"]["compliant"], 1)
native_row = self.report()["series"]["independent:artificial-reviewer-a"]["runs"][1]
self.assertEqual(native_row["b3_native_coverage"]["stage_attempt_counts"]["selection"], 1)
def test_malformed_comparison_is_invalid_not_a_synthesized_vote(self):
descriptor = self.iteration["comparisons"][0]["attempts"][0]
raw = b"Artificial invalid fixture.tie"
self.replace(descriptor["response_record_id"], raw)
descriptor["response_sha256"] = audit.sealing.sha(raw)
env = self.envelope()
env["records"][0]["choice"] = None
env["result"] = self.decision("s0", "s1", complete=False)
self.update_selection(env)
self.iteration["next_incumbent_id"] = self.row["final_incumbent_id"] = "s0"
self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"]
self.reseal()
self.f.prepare()
stats = self.native(self.report())["b3_native_coverage"]
self.assertEqual(stats["incomplete_iteration_count"], 1)
self.assertEqual(stats["incomplete_selection_retention_count"], 1)
self.assertEqual(stats["rejected_candidate_count"], 0)
def test_interrupted_invalid_proposal_retains_old_incumbent_and_failure(self):
descriptor = self.iteration["generation_attempts"][0]
raw = b"Artificial malformed XML fixture, not a proposal."
self.replace(descriptor["response_record_id"], raw)
descriptor["response_sha256"] = audit.sealing.sha(raw)
keep = {self.initial_id, descriptor["attempt_id"]}
self.run["attempts"] = [a for a in self.run["attempts"] if a["attempt_id"] in keep]
next(a for a in self.run["attempts"] if a["attempt_id"] == descriptor["attempt_id"])["status"] = "failed"
self.row["packets"] = [p for p in self.row["packets"] if p["attempt_id"] in keep]
self.iteration.update(generation_attempt_id=None, worker_attempts=[], artifact_attempt_id=None, next_incumbent_id="s0")
for rnd in self.iteration["comparisons"]:
rnd.update(attempts=[], vote_attempt_id=None)
self.iteration["selection"].update(state="not_executed", record_id=None, sha256=None)
self.state("s1").update(prompt=None, artifact_record_id=None, artifact_sha256=None)
self.row["final_incumbent_id"] = "s0"
self.run["endpoint_selection"]["artifact_id"] = self.state("s0")["artifact_record_id"]
self.reseal()
self.f.prepare()
stats = self.native(self.report())["b3_native_coverage"]
self.assertEqual(stats["incomplete_iteration_count"], 1)
native_row = self.report()["series"]["independent:artificial-reviewer-a"]["runs"][1]
self.assertEqual(native_row["b3_native_coverage"]["attempt_status_counts"]["failed"], 1)
self.assertEqual(native_row["b3_native_coverage"]["iterations"][0]["generation_invalid_response_count"], 1)
def test_invalid_record_or_missing_vote_must_keep_incumbent_with_null_counts(self):
env = self.envelope()
env["records"] = env["records"][:3]
result = self.decision("s0", "s1", complete=False)
self.assertFalse(b3.selection_consistent(env["records"], result, "s0", "s1"))
result["candidate_votes"] = 3
with self.assertRaisesRegex(ValueError, "four-vote rule"):
b3.selection_consistent(env["records"], result, "s0", "s1")
result = self.decision("s0", "s1", complete=False)
duplicated = [env["records"][0]]*4
self.assertFalse(b3.selection_consistent(duplicated, result, "s0", "s1"))
def test_packet_hash_response_provenance_and_duplicate_json_are_checked(self):
original = self.row["packets"][1]["sha256"]
self.row["packets"][1]["sha256"] = "0"*64
with self.assertRaisesRegex(ValueError, "hash mismatch"):
self.f.prepare()
self.row["packets"][1]["sha256"] = original
desc = self.iteration["generation_attempts"][0]
desc["response_sha256"] = "0"*64
with self.assertRaisesRegex(ValueError, "hash mismatch"):
self.f.prepare()
raw = self.render("worker", dict(CANDIDATE_PROMPT_JSON="fixture", TASK_PACKET_JSON=self.task))
raw = raw.replace(b'"fixture"', b'{"id":1,"id":2}')
with self.assertRaises(ValueError):
b3.decode_input(raw, "worker", self.f.manifest["bounds"])
def test_xml_escaping_does_not_forward_analysis_and_typed_selection_is_strict(self):
parsed = b3.xml_response(b"fixturefixtureA & B", "generation")
self.assertEqual(parsed["prompt"], "A & B")
self.assertIsNone(b3.xml_response(b"x A ", "comparison"))
env = self.envelope()
env["result"]["comparison_complete"] = 1
with self.assertRaises(ValueError):
b3.selection_consistent(env["records"], env["result"], "s0", "s1")
def test_independent_and_adjudicated_forms_remain_separate(self):
self.f.prepare()
forms = [self.base.completed(0), self.base.completed(1, "independent-2.blank.json"), self.base.completed(0, "adjudicated.blank.json")]
for row in forms[-1]["ratings"]:
row.update(independent_judgments=[0, 1], resolution_note="Artificial resolution fixture; no study judgment.")
paths = [self.f.write_form(form, str(i)+".json") for i, form in enumerate(forms)]
report = self.f.summarize(paths)
self.assertEqual(self.native(report)["counts"]["compliant"], 1)
self.assertEqual(report["series"]["independent:artificial-reviewer-b"]["by_condition"]["B3"]["counts"]["noncompliant"], 1)
self.assertEqual(report["series"]["adjudicated"]["by_condition"]["B3"]["counts"]["compliant"], 1)
def test_missing_declared_candidate_state_is_unknown_not_reconstructed(self):
self.state("s1").update(prompt=None, artifact_record_id=None, artifact_sha256=None)
self.f.prepare()
self.assertEqual(self.native(self.report())["counts"]["unassessed"], 1)
def test_explicitly_unproduced_generation_cannot_feed_worker(self):
self.iteration["generation_attempts"][0].update(response_state="not_produced", response_record_id=None, response_sha256=None)
with self.assertRaisesRegex(ValueError, "unproduced generation"):
self.f.prepare()
def test_missing_next_and_final_links_are_unknown_not_inferred(self):
self.iteration["next_incumbent_id"] = self.row["final_incumbent_id"] = None
self.f.prepare()
self.assertEqual(self.native(self.report())["counts"]["unassessed"], 1)
def test_missing_archived_endpoint_keeps_failed_run_in_denominator(self):
self.run["run_status"] = "failed"
self.run["endpoint_selection"]["artifact_id"] = None
self.reseal()
self.f.prepare()
report = self.report()
self.assertEqual(self.native(report)["counts"]["unassessed"], 1)
self.assertEqual(report["series"]["independent:artificial-reviewer-a"]["overall"]["assigned_run_denominator"], 2)
if __name__ == "__main__":
unittest.main()