"""Temporary artificial metadata, Unicode spans and categorical fixtures only.""" import contextlib import copy from datetime import datetime, timezone import io import json from pathlib import Path import sys import unittest from unittest import mock import assess_claims as support sys.path.insert(0, str(Path(__file__).resolve().parent.parent/"run-sealing")) import test_seal_runs as fixtures class ClaimSupportTests(unittest.TestCase): def setUp(self): self.f = fixtures.SealingTests() self.f.setUp() self.addCleanup(self.f.doCleanups) f = self.f # Artificial text and spans, not answers or claims about any study task. self.text = "\ufeffArtificial Unicode fixture.\r\nα=7.\r\nβ=9.\r\n" content = self.text.encode("utf-8") (f.root/"fixture-a-endpoint.source").write_bytes(content) f.contents["fixture-a-endpoint"] = content f.source["records"] = [r for r in f.source["records"] if r["record_id"] != "fixture-usage"] for run in f.source["runs"]: run["record_ids"].remove("fixture-usage") f.source["conditions"].append(dict(condition_id="fixture-second-condition", repetition_ids=["fixture-repetition"])) f.source["runs"][1]["condition_id"] = "fixture-second-condition" f.set_availability() for run in f.source["runs"]: run["record_availability"]["usage_records"] = dict(state="unavailable", note="Artificial usage unavailable") f.seal() commit = json.loads((f.output/"packet-build-manifest.json").read_bytes()) self.manifest = dict(claim_manifest_version="1.0", assessment_id="artificial-support-assessment", independent_rater_ids=["source-rater-a", "source-rater-b"], adjudicator_id="source-adjudicator", archives=[dict(archive_id="fixture-archive", path=f.output.name)], artifacts=[]) self.inventories, self.inventory_paths = {}, {} for index, run in enumerate(commit["runs"]): row = {k: run[k] for k in ("run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id")} row.update(archive_id="fixture-archive", seal_record_id=commit["seal_record_id"], artifact_id=run["final_artifact"]["artifact_id"], artifact_sha256=run["final_artifact"]["sha256"], inventory=None, unavailability_reason=None) claims = [] if index == 0: for label, text in (("human-alpha", "α=7."), ("human-beta", "β=9.")): start = self.text.index(text) claims.append(dict(claim_id=label, start=start, end=start+len(text), text=text)) inventory = {k: row[k] for k in ("artifact_id", "artifact_sha256", "task_id", "task_version", "rubric_version")} inventory.update(claim_inventory_version="1.0", enumeration_status="reviewed", enumerator_id="fixture-enumerator-private", completed_at_utc=self.now(), note="Artificial enumeration note: "+row["condition_id"], claims=claims) self.inventories[row["run_id"]] = inventory self.inventory_paths[row["run_id"]] = f.root/f"inventory-{index}.json" self.manifest["artifacts"].append(row) self.manifest_path = f.root/"claim-selection.json" self.raters, self.coordinator = f.root/"rater-claims", f.root/"private-key" self.key_path = self.coordinator/"claim-key.json" self.save() def now(self): return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") def save(self): for row in self.manifest["artifacts"]: rid = row["run_id"] data = support.sealing.json_bytes(self.inventories[rid]) self.inventory_paths[rid].write_bytes(data) row["inventory"] = dict(path=self.inventory_paths[rid].name, sha256=support.sealing.sha(data)) self.manifest_path.write_bytes(support.sealing.json_bytes(self.manifest)) def build(self, seed=0, raters=None, coordinator=None): support.build(self.manifest_path, raters or self.raters, coordinator or self.coordinator, self.f.registry_path, seed) return json.loads(((coordinator or self.coordinator)/"claim-key.json").read_bytes()) def form_path(self, alias): return self.raters/alias/"rating-form.json" def fill(self, key, alias, judgments, adjudicate=False): path = self.form_path(alias) form = json.loads(path.read_bytes()) for row, judgment in zip(form["ratings"], judgments): if judgment is None: continue row.update(judgment=judgment, reason="Artificial categorical fixture, no substantive judgment.", evidence_locations=["Artificial fixed packet location"], completed_at_utc=self.now()) if adjudicate: basis = {} for rater in key["independent_rater_aliases"].values(): original = json.loads(self.form_path(rater).read_bytes()) basis[rater] = next(r["judgment"] for r in original["ratings"] if r["claim_id"] == row["claim_id"]) row.update(independent_judgments=basis, resolution_note="Artificial resolution fixture.") form["locked_at_utc"] = self.now() path.write_bytes(support.sealing.json_bytes(form)) return path def summary(self, paths=()): return support.summarize(self.key_path, paths, self.f.registry_path) def test_byte_preservation_blinded_fields_and_blank_judgments(self): key = self.build() for spec in key["forms"]: root = self.raters/spec["assessor_id"] for artifact in key["artifacts"]: source = next(r for r in self.manifest["artifacts"] if r["run_id"] == artifact["source_run_id"]) self.assertEqual((root/"artifacts"/(artifact["artifact_id"]+".txt")).read_bytes(), self.f.contents[source["artifact_id"]]) inventory_path = root/"claims"/(artifact["artifact_id"]+".json") self.assertEqual(support.sealing.sha(inventory_path.read_bytes()), artifact["inventory_sha256"]) if artifact["claims"]: claim = json.loads(inventory_path.read_bytes())["claims"][0] self.assertEqual(self.text[claim["start"]:claim["end"]], claim["text"]) self.assertTrue(claim["claim_id"].startswith("claim-")) for path in root.rglob("*"): if path.is_file() and "artifacts" not in path.parts: content = path.read_bytes() for forbidden in (b"condition_id", b"fixture-condition", b"fixture-run-", b"fixture-enumerator-private", b"human-alpha", b"artificial_configuration"): self.assertNotIn(forbidden, content) form = json.loads((root/"rating-form.json").read_bytes()) self.assertTrue(all(r["judgment"] is None for r in form["ratings"])) self.assertTrue(all("source_packet_sha256" in r for r in form["ratings"])) def test_blank_forms_missing_coverage_and_reviewed_empty_inventory(self): key = self.build() report = self.summary([self.form_path(s["assessor_id"]) for s in key["forms"]]) rater = next(iter(key["independent_rater_aliases"].values())) group = report["independent"][rater]["overall"] self.assertEqual(group["missing_judgments"], 2) self.assertEqual(group["assessed_claims"], 0) self.assertEqual(group["reviewed_no_claim_artifacts"], 1) self.assertFalse(group["declared_coverage_complete"]) self.assertIsNone(group["unsupported_claim_rate"]) empty = report["independent"][rater]["by_condition"]["fixture-second-condition"] self.assertTrue(empty["declared_coverage_complete"]) self.assertIsNone(empty["unsupported_claim_rate"]) self.assertIsNone(report["independent_rater_agreement"]["overall"]["raw_agreement"]) def test_complete_counts_series_separation_and_agreement(self): key = self.build() a, b = key["independent_rater_aliases"].values() p1 = self.fill(key, a, ["unsupported", "supported"]) p2 = self.fill(key, b, ["supported", "supported"]) report = self.summary([p1, p2]) group = report["independent"][a]["overall"] self.assertEqual((group["unsupported_claims"], group["assessed_claims"], group["missing_judgments"]), (1, 2, 0)) self.assertEqual(group["unsupported_claim_rate"], .5) self.assertEqual(report["independent"][b]["overall"]["unsupported_claim_rate"], 0) self.assertIsNone(report["adjudicated"]["overall"]["unsupported_claim_rate"]) agreement = report["independent_rater_agreement"]["overall"] self.assertEqual((agreement["agreeing_claims"], agreement["jointly_assessed_claim_denominator"], agreement["raw_agreement"]), (1, 2, .5)) def test_partial_judgments_counts_not_imputed_and_joint_denominator(self): key = self.build() a, b = key["independent_rater_aliases"].values() p1 = self.fill(key, a, ["unsupported", None]) p2 = self.fill(key, b, ["unsupported", "supported"]) report = self.summary([p1, p2]) group = report["independent"][a]["overall"] self.assertEqual((group["unsupported_claims"], group["assessed_claims"], group["missing_judgments"]), (1, 1, 1)) self.assertIsNone(group["unsupported_claim_rate"]) self.assertEqual(report["independent_rater_agreement"]["overall"]["jointly_assessed_claim_denominator"], 1) self.assertEqual(report["independent_rater_agreement"]["overall"]["raw_agreement"], 1) def test_unfinished_inventory_with_all_current_claims_rated_still_incomplete(self): self.inventories["fixture-run-a"].update(enumeration_status="unfinished", completed_at_utc=None) self.save() key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, ["supported", "supported"]) group = self.summary([path])["independent"][a]["overall"] self.assertEqual(group["assessed_claims"], 2) self.assertEqual(group["unfinished_inventories"], 1) self.assertEqual(group["missing_judgments"], 0) self.assertIsNone(group["unsupported_claim_rate"]) def test_empty_unfinished_inventory_is_not_a_reviewed_no_claim_declaration(self): self.inventories["fixture-run-a"].update(enumeration_status="unfinished", completed_at_utc=None, claims=[]) self.save() self.build() group = self.summary()["adjudicated"]["overall"] self.assertEqual(group["unfinished_inventories"], 1) self.assertEqual(group["reviewed_no_claim_artifacts"], 1) self.assertEqual(group["provided_claims"], 0) self.assertFalse(group["declared_coverage_complete"]) self.assertIsNone(group["unsupported_claim_rate"]) def test_unavailable_inventory_is_not_reviewed_empty(self): self.manifest["artifacts"][0].update(inventory=None, unavailability_reason="Artificial unavailable inventory") self.manifest_path.write_bytes(support.sealing.json_bytes(self.manifest)) self.inventory_paths["fixture-run-a"].write_bytes(b"This undeclared file must never be read.") key = self.build() a = next(iter(key["independent_rater_aliases"].values())) group = self.summary()["independent"][a]["overall"] self.assertEqual((group["unavailable_inventories"], group["reviewed_no_claim_artifacts"]), (1, 1)) self.assertFalse(group["declared_coverage_complete"]) self.assertIsNone(group["artifacts"][0]["enumerated_claim_count"]) self.assertIsNone(group["unsupported_claim_rate"]) def test_all_reviewed_no_claims_has_null_zero_denominator_not_zero_errors(self): self.inventories["fixture-run-a"]["claims"] = [] self.save() key = self.build() group = self.summary()["adjudicated"]["overall"] self.assertTrue(group["declared_coverage_complete"]) self.assertEqual(group["reviewed_no_claim_artifacts"], 2) self.assertEqual(group["assessed_claims"], 0) self.assertIsNone(group["unsupported_claim_rate"]) self.assertIsNone(group["judgment_coverage"]["rate"]) def test_invalid_spans_and_duplicate_claims_rejected(self): original = copy.deepcopy(self.inventories["fixture-run-a"]) cases = ( (lambda i: i["claims"].append(copy.deepcopy(i["claims"][0])), "duplicate claim identity"), (lambda i: i["claims"].append(dict(i["claims"][0], claim_id="another-id")), "duplicate claim span"), (lambda i: i["claims"][0].update(start=True), "invalid claim span bounds"), (lambda i: i["claims"][0].update(end=10000), "invalid claim span bounds"), (lambda i: i["claims"][0].update(end=i["claims"][0]["start"]), "invalid claim span bounds"), (lambda i: i["claims"][0].update(text="does not match"), "does not match exact artifact"), ) for change, error in cases: with self.subTest(error=error): self.inventories["fixture-run-a"] = copy.deepcopy(original) change(self.inventories["fixture-run-a"]) self.save() with self.assertRaisesRegex(support.sealing.ValidationError, error): self.build() self.assertFalse(self.raters.exists()) def test_inventory_hash_identity_version_and_review_time_rejected(self): original = copy.deepcopy(self.inventories["fixture-run-a"]) for change, error in ((lambda i: i.update(artifact_sha256="0"*64), "identity/hash/version"), (lambda i: i.update(task_version="2.0"), "identity/hash/version"), (lambda i: i.update(claim_inventory_version="2.0"), "incompatible"), (lambda i: i.update(completed_at_utc="2000-01-01T00:00:00Z"), "follow sealing")): self.inventories["fixture-run-a"] = copy.deepcopy(original) change(self.inventories["fixture-run-a"]) self.save() with self.assertRaisesRegex(support.sealing.ValidationError, error): self.build() self.inventories["fixture-run-a"] = original self.save() self.manifest["artifacts"][0]["inventory"]["sha256"] = "0"*64 self.manifest_path.write_bytes(support.sealing.json_bytes(self.manifest)) with self.assertRaisesRegex(support.sealing.ValidationError, "source hash mismatch"): self.build() def test_adjudication_requires_both_independent_locked_judgments(self): key = self.build() a, b = key["independent_rater_aliases"].values() p1 = self.fill(key, a, ["unsupported", "supported"]) p2 = self.fill(key, b, ["supported", "supported"]) pa = self.fill(key, key["adjudicator_alias"], ["supported", "supported"], adjudicate=True) with self.assertRaisesRegex(support.sealing.ValidationError, "both independent"): self.summary([pa, p1]) report = self.summary([pa, p2, p1]) self.assertEqual(report["adjudicated"]["overall"]["unsupported_claim_rate"], 0) self.assertEqual(report["independent"][a]["overall"]["unsupported_claim_rate"], .5) form = json.loads(pa.read_bytes()) form["ratings"][0]["independent_judgments"][a] = "supported" pa.write_bytes(support.sealing.json_bytes(form)) with self.assertRaisesRegex(support.sealing.ValidationError, "stale adjudication"): self.summary([p1, p2, pa]) def test_rating_duplicate_and_hash_version_judgment_mismatches(self): key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.form_path(a) original = path.read_bytes() with self.assertRaisesRegex(support.sealing.ValidationError, "duplicate assessment series"): self.summary([path, path]) changes = ((lambda f: f["ratings"].append(copy.deepcopy(f["ratings"][0])), "duplicate assessed claim"), (lambda f: f["ratings"][0].update(source_packet_sha256="0"*64), "identity or hash mismatch"), (lambda f: f["ratings"][0].update(inventory_sha256="0"*64), "identity or hash mismatch"), (lambda f: f["ratings"][0].update(start=0), "identity or hash mismatch"), (lambda f: f["ratings"][0].update(start=False), "invalid rating claim span types"), (lambda f: f.update(claim_form_version="2.0"), "incompatible"), (lambda f: f["ratings"][0].update(judgment=True), "invalid human support judgment"), (lambda f: f["ratings"][0].update(condition_id="excluded"), "unexpected/missing fields")) for change, error in changes: form = json.loads(original) change(form) path.write_bytes(support.sealing.json_bytes(form)) with self.assertRaisesRegex(support.sealing.ValidationError, error): self.summary([path]) def test_judgments_require_post_inventory_timestamps_and_locked_forms(self): key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, ["supported", "unsupported"]) original = path.read_bytes() for change, error in ((lambda f: f.update(locked_at_utc=None), "precede form lock"), (lambda f: f["ratings"][0].update(completed_at_utc="2000-01-01T00:00:00Z"), "follow sealing/inventory review"), (lambda f: f["ratings"][0].update(evidence_locations=[]), "needs evidence locations")): form = json.loads(original) change(form) path.write_bytes(support.sealing.json_bytes(form)) with self.assertRaisesRegex(support.sealing.ValidationError, error): self.summary([path]) def test_duplicate_selections_unsealed_and_not_final_artifacts_rejected(self): original = copy.deepcopy(self.manifest) self.manifest["artifacts"].append(copy.deepcopy(self.manifest["artifacts"][0])) self.manifest_path.write_bytes(support.sealing.json_bytes(self.manifest)) with self.assertRaisesRegex(support.sealing.ValidationError, "duplicate selection"): self.build() self.manifest = original self.manifest["artifacts"][0]["artifact_id"] = "fixture-a-other" self.manifest_path.write_bytes(support.sealing.json_bytes(self.manifest)) with self.assertRaisesRegex(support.sealing.ValidationError, "final artifact identity/hash mismatch"): self.build() self.manifest["artifacts"][0]["artifact_id"] = "fixture-a-endpoint" self.save() (self.f.output/"packet-build-manifest.json").unlink() with self.assertRaisesRegex(support.sealing.ValidationError, "unsealed"): self.build() def test_copied_inventory_binding_and_original_inputs_not_needed(self): key = self.build() for path in self.inventory_paths.values(): path.unlink() self.manifest_path.unlink() self.summary() copied = self.coordinator/key["inventory_copies"][0]["path"] copied.write_bytes(copied.read_bytes()+b"corruption") with self.assertRaisesRegex(support.sealing.ValidationError, "source hash mismatch"): self.summary() def test_seeded_presentation_with_fresh_opaque_ids(self): first = self.build(seed=42) second = self.build(seed=42, raters=self.f.root/"second-raters", coordinator=self.f.root/"second-key") def orders(key): lookup = {a["artifact_id"]: a["source_run_id"] for a in key["artifacts"]} return [[lookup[a] for a in f["artifact_order"]] for f in key["forms"]] self.assertEqual(orders(first), orders(second)) self.assertNotEqual(first["batch_id"], second["batch_id"]) def test_read_only_inputs_overwrite_refusal_and_cli(self): def snapshot(root): return {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file()} before = snapshot(self.f.output) with self.assertRaisesRegex(support.sealing.ValidationError, "outside sealed archives"): self.build(raters=self.f.output/"new-raters") with self.assertRaisesRegex(support.sealing.ValidationError, "disjoint"): self.build(raters=self.f.root/"same", coordinator=self.f.root/"same") key = self.build() rater_bytes, coordinator_bytes = snapshot(self.raters), snapshot(self.coordinator) with self.assertRaisesRegex(support.sealing.ValidationError, "existing output"): self.build() stdout, stderr = io.StringIO(), io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = support.main(["summarize", "--key", str(self.key_path), "--registry", str(self.f.registry_path)]) self.assertEqual(code, 0) self.assertEqual(json.loads(stdout.getvalue())["claim_support_report_version"], "1.0") self.assertEqual(before, snapshot(self.f.output)) self.assertEqual(rater_bytes, snapshot(self.raters)) self.assertEqual(coordinator_bytes, snapshot(self.coordinator)) def test_failed_writes_do_not_publish_key_or_success_and_duplicate_json_rejected(self): stdout, stderr = io.StringIO(), io.StringIO() with mock.patch.object(support.packets, "write_bytes", side_effect=OSError("artificial write failure")): with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = support.main(["build", "--manifest", str(self.manifest_path), "--rater-output", str(self.raters), "--coordinator-output", str(self.coordinator), "--registry", str(self.f.registry_path)]) self.assertEqual(code, 2) self.assertFalse(self.key_path.exists()) self.assertEqual(stdout.getvalue(), "") with self.assertRaisesRegex(support.sealing.ValidationError, "duplicate JSON key"): support.sealing.parse_json(b'{"claim_id":"a","claim_id":"b"}', "artificial duplicate fixture") if __name__ == "__main__": unittest.main()