"""Temporary artificial byte/metadata/categorical fixtures; no study findings.""" import contextlib import copy from datetime import datetime, timezone import io import json from pathlib import Path import sys import unittest import pairwise_preferences as preference sys.path.insert(0, str(Path(__file__).resolve().parent.parent/"run-sealing")) import test_seal_runs as fixtures class PairwiseTests(unittest.TestCase): def setUp(self): self.f = fixtures.SealingTests() self.f.setUp() self.addCleanup(self.f.doCleanups) f = self.f self.target, self.control = "fixture-condition-target", "fixture-condition-control" self.repetitions = ["fixture-rep-1", "fixture-rep-2"] f.source["conditions"] = [dict(condition_id=c, repetition_ids=self.repetitions) for c in (self.target, self.control)] f.source["records"] = [r for r in f.source["records"] if r["record_id"] in ("fixture-config", "fixture-prompt-a")] f.source["runs"] = [] pairs = [] for index, task in enumerate(f.registry["tasks"]): for rep in self.repetitions: endpoints = {} for role, cid in (("target", self.target), ("comparator", self.control)): rid = f"fixture-run-{index}-{rep}-{role}" aid = f"fixture-artifact-{index}-{rep}-{role}" content = ("Artificial byte fixture only; no task answer. Café.\r\n" "Intrinsic method disclosure is preserved.\r\n"+aid+"\r\n").encode() f.add_record(aid, "artifact", content) run = dict(run_id=rid, condition_id=cid, task_id=task["task_id"], task_version="1.0", rubric_version="1.0", repetition_id=rep, run_status="completed", ended_at_utc=None, endpoint_selection=dict(artifact_id=aid, basis="Explicit artificial endpoint"), record_ids=["fixture-config", "fixture-prompt-a", aid], record_availability={}, attempts=[dict(attempt_id="attempt-"+rid, phase="adaptation", role="worker", status="succeeded", retry_of=None, started_at_utc=None, ended_at_utc=None, record_ids=["fixture-prompt-a"])]) f.source["runs"].append(run) endpoints[role] = dict(archive_id="fixture-archive", seal_record_id=None, run_id=rid, condition_id=cid, artifact_id=aid, sha256=preference.sealing.sha(content)) pairs.append(dict(pair_id=f"fixture-pair-{index:02d}-{rep}", task_id=task["task_id"], task_version="1.0", rubric_version="1.0", repetition_id=rep, **endpoints)) f.set_availability() f.seal() seal = json.loads((f.output/"packet-build-manifest.json").read_bytes())["seal_record_id"] for pair in pairs: for role in ("target", "comparator"): pair[role]["seal_record_id"] = seal self.manifest = dict(pair_manifest_version="1.0", study_id="artificial-pair-batch", target_condition_id=self.target, comparator_condition_id=self.control, repetition_ids=self.repetitions, independent_rater_ids=["source-rater-a", "source-rater-b"], adjudicator_id="source-adjudicator", archives=[dict(archive_id="fixture-archive", path=f.output.name)], pairs=pairs) self.manifest_path = f.root/"pair-selection.json" self.rater_dir, self.coord_dir = f.root/"rater-packets", f.root/"coordinator-key" self.key_path = self.coord_dir/"comparison-key.json" self.save_manifest() def save_manifest(self): self.manifest_path.write_bytes(preference.sealing.json_bytes(self.manifest)) def build(self, seed=7, rater_dir=None, coord_dir=None): preference.build(self.manifest_path, rater_dir or self.rater_dir, coord_dir or self.coord_dir, self.f.registry_path, seed) return json.loads(((coord_dir or self.coord_dir)/"comparison-key.json").read_bytes()) def form_path(self, alias): return self.rater_dir/alias/"rating-form.json" def fill(self, key, alias, outcome, adjudicate=False): path = self.form_path(alias) form = json.loads(path.read_bytes()) comparisons = {c["comparison_id"]: c for c in key["comparisons"]} for row in form["ratings"]: comparison = comparisons[row["comparison_id"]] for dimension in preference.DIMENSIONS: result = outcome(comparison, dimension) if result is None: continue winner = self.target if result == "win" else self.control judgment = "tie" if result == "tie" else ("left_win" if comparison["left_condition_id"] == winner else "right_win") value = row[dimension] value.update(judgment=judgment, reason="Artificial categorical fixture only.", evidence_locations=["Artificial left/right text fixture location"], completed_at_utc=datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")) if adjudicate: basis = {} for rater in key["independent_rater_aliases"].values(): original = json.loads(self.form_path(rater).read_bytes()) original_row = next(r for r in original["ratings"] if r["comparison_id"] == row["comparison_id"]) basis[rater] = original_row[dimension]["judgment"] value.update(independent_judgments=basis, resolution_note="Artificial resolution fixture.") form["locked_at_utc"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") path.write_bytes(preference.sealing.json_bytes(form)) return path def summarize(self, paths=(), seed=11): return preference.summarize(self.key_path, paths, self.f.registry_path, replicates=100, seed=seed) def test_byte_preservation_opaque_metadata_separation_and_blank_forms(self): key = self.build() for spec in key["forms"]: root = self.rater_dir/spec["assessor_id"] for comparison in key["comparisons"]: for side in ("left", "right"): rid = comparison[side+"_run_id"] source = next(r for r in self.f.source["runs"] if r["run_id"] == rid) aid = source["endpoint_selection"]["artifact_id"] self.assertEqual((root/"artifacts"/(comparison[side+"_artifact_id"]+".txt")).read_bytes(), self.f.contents[aid]) for path in root.rglob("*"): if path.is_file(): self.assertNotIn(self.target, str(path.relative_to(root))) if "artifacts" not in path.parts: content = path.read_bytes() self.assertNotIn(self.target.encode(), content) self.assertNotIn(b"fixture-run-", content) self.assertNotIn(b"artificial_configuration", content) self.assertNotIn(b"condition_id", content) form = json.loads((root/"rating-form.json").read_bytes()) self.assertTrue(all(row[d]["judgment"] is None for row in form["ratings"] for d in preference.DIMENSIONS)) self.assertEqual((self.coord_dir/"source-manifest.json").read_bytes(), self.manifest_path.read_bytes()) def test_seeded_orientation_not_seeded_opaque_ids(self): first = self.build() second = self.build(rater_dir=self.f.root/"rater-two", coord_dir=self.f.root/"key-two") self.assertEqual([c["left_condition_id"] for c in first["comparisons"]], [c["left_condition_id"] for c in second["comparisons"]]) self.assertNotEqual(first["batch_id"], second["batch_id"]) self.assertEqual(set(c["left_condition_id"] for c in first["comparisons"]), {self.target, self.control}) def test_separate_dimensions_orientation_and_agreement(self): key = self.build() raters = list(key["independent_rater_aliases"].values()) p1 = self.fill(key, raters[0], lambda c, d: "win" if d == "usefulness" else "loss") p2 = self.fill(key, raters[1], lambda c, d: "win" if d == "usefulness" else "tie") report = self.summarize([p1, p2]) useful = report["independent"][raters[0]]["usefulness"] self.assertEqual(useful["counts_by_condition"][self.target], dict(win=24, tie=0, loss=0)) self.assertEqual(useful["counts_by_condition"][self.control], dict(win=0, tie=0, loss=24)) self.assertEqual(useful["preference_estimate"], 1) self.assertEqual(report["independent"][raters[0]]["clarity"]["preference_estimate"], 0) self.assertIsNone(report["adjudicated"]["usefulness"]["preference_estimate"]) self.assertEqual(report["agreement"]["independent_raters"]["usefulness"]["raw_agreement"], 1) self.assertEqual(report["agreement"]["independent_raters"]["clarity"]["raw_agreement"], 0) self.assertEqual(report["agreement"]["independent_raters"]["clarity"]["jointly_assessed_pair_denominator"], 24) def test_repetition_task_family_weighting_and_bootstrap_reproducibility(self): key = self.build() rater = next(iter(key["independent_rater_aliases"].values())) tasks = self.f.registry["families"][0]["task_ids"] def values(c, dimension): if c["task_id"] not in tasks: return "tie" return "win" if c["task_id"] == tasks[0] and c["repetition_id"] == self.repetitions[0] else "loss" path = self.fill(key, rater, values) report = self.summarize([path]) result = report["independent"][rater]["usefulness"] self.assertEqual(result["preference_estimate"], .375) groups = {f["family_id"]: ([.5, 0, 0, 0] if index == 0 else [.5]*4) for index, f in enumerate(self.f.registry["families"])} self.assertEqual(result["confidence_interval_95"], preference.bootstrap.bootstrap_interval(groups, 100, 11)) self.assertEqual(report, self.summarize([path])) other_seed = self.summarize([path], seed=2) self.assertEqual(other_seed["bootstrap"]["seed"], 2) self.assertEqual(other_seed["independent"][rater]["usefulness"]["confidence_interval_95"], preference.bootstrap.bootstrap_interval(groups, 100, 2)) self.assertEqual(len(result["task_clusters"]), 12) self.assertTrue(all(t["assessed_repetitions"] == 2 for t in result["task_clusters"])) def test_missing_judgments_withhold_only_affected_series_dimension(self): key = self.build() rater = next(iter(key["independent_rater_aliases"].values())) missing = key["comparisons"][0]["comparison_id"] path = self.fill(key, rater, lambda c, d: None if c["comparison_id"] == missing and d == "usefulness" else "tie") report = self.summarize([path]) self.assertIsNone(report["independent"][rater]["usefulness"]["preference_estimate"]) self.assertIsNone(report["independent"][rater]["usefulness"]["confidence_interval_95"]) self.assertEqual(report["independent"][rater]["usefulness"]["missing_judgment_comparison_ids"], [missing]) self.assertEqual(report["independent"][rater]["clarity"]["preference_estimate"], .5) self.assertIsNone(report["agreement"]["independent_raters"]["clarity"]["raw_agreement"]) def test_joint_agreement_denominator_excludes_missing_judgments(self): key = self.build() raters = list(key["independent_rater_aliases"].values()) missing = key["comparisons"][0]["comparison_id"] p1 = self.fill(key, raters[0], lambda c, d: "tie") p2 = self.fill(key, raters[1], lambda c, d: None if c["comparison_id"] == missing else "tie") report = self.summarize([p1, p2]) value = report["agreement"]["independent_raters"]["usefulness"] self.assertEqual(value, dict(agreeing_pairs=23, jointly_assessed_pair_denominator=23, missing_joint_judgment_pairs=1, raw_agreement=1)) def test_bootstrap_family_weights_are_equal_not_proportional_to_cluster_count(self): # Direct mathematical fixture for the reused general bootstrap helper; # this is not a task registry or a study estimate. groups = {"artificial-a": [1.], "artificial-b": [0., 0.], "artificial-c": [0., 0., 0.]} self.assertEqual(preference.bootstrap.equal_family_mean(groups), 1/3) self.assertEqual(preference.bootstrap.bootstrap_interval(groups, 100, 11), [1/3, 1/3]) def test_missing_declared_pair_cells_withhold_aggregates(self): self.manifest["pairs"] = self.manifest["pairs"][:1] self.save_manifest() key = self.build() rater = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, rater, lambda c, d: "win") result = self.summarize([path])["independent"][rater]["usefulness"] self.assertEqual(result["expected_pairs"], 24) self.assertEqual(len(result["absent_pair_cells"]), 23) self.assertEqual(result["assessed_pairs"], 1) self.assertIsNone(result["preference_estimate"]) def test_adjudication_stays_separate_and_requires_original_ratings(self): key = self.build() raters = list(key["independent_rater_aliases"].values()) p1 = self.fill(key, raters[0], lambda c, d: "win") p2 = self.fill(key, raters[1], lambda c, d: "loss") pa = self.fill(key, key["adjudicator_alias"], lambda c, d: "tie", adjudicate=True) with self.assertRaisesRegex(preference.sealing.ValidationError, "both independent"): self.summarize([p1, pa]) report = self.summarize([pa, p1, p2]) self.assertEqual(report["adjudicated"]["usefulness"]["preference_estimate"], .5) self.assertEqual(report["independent"][raters[0]]["usefulness"]["preference_estimate"], 1) self.assertEqual(report["agreement"]["adjudicated_vs_independent"][raters[0]]["usefulness"]["raw_agreement"], 0) form = json.loads(pa.read_bytes()) form["ratings"][0]["usefulness"]["independent_judgments"][raters[0]] = "tie" pa.write_bytes(preference.sealing.json_bytes(form)) with self.assertRaisesRegex(preference.sealing.ValidationError, "stale adjudication basis"): self.summarize([p1, p2, pa]) def test_locked_form_and_post_seal_completion_are_required(self): key = self.build() rater = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, rater, lambda c, d: "tie") original = path.read_bytes() for change in (lambda f: f.update(locked_at_utc=None), lambda f: f["ratings"][0]["clarity"].update(completed_at_utc="2000-01-01T00:00:00Z")): form = json.loads(original) change(form) path.write_bytes(preference.sealing.json_bytes(form)) with self.assertRaisesRegex(preference.sealing.ValidationError, "follow sealing and precede form lock"): self.summarize([path]) def test_duplicate_pairs_and_mismatched_endpoints_rejected(self): original = copy.deepcopy(self.manifest) changes = ( (lambda m: m["pairs"].append(copy.deepcopy(m["pairs"][0])), "duplicate pair"), (lambda m: m["pairs"][0]["target"].update(sha256="0"*64), "identity/hash mismatch"), (lambda m: m["pairs"][0]["comparator"].update(condition_id=self.target), "condition identity mismatch"), (lambda m: m["pairs"][0]["comparator"].update(run_id=m["pairs"][1]["comparator"]["run_id"]), "match task, versions, and repetition"), (lambda m: m["pairs"][0].update(rubric_version="2.0"), "incompatible"), ) for change, error in changes: with self.subTest(error=error): self.manifest = copy.deepcopy(original) change(self.manifest) self.save_manifest() with self.assertRaisesRegex(preference.sealing.ValidationError, error): self.build() self.assertFalse(self.rater_dir.exists()) def test_unsealed_incompatible_and_corrupt_archives_rejected(self): path = self.f.output/"packet-build-manifest.json" original = path.read_bytes() path.unlink() with self.assertRaisesRegex(preference.sealing.ValidationError, "unsealed"): self.build() packet = json.loads(original) packet["build_manifest_version"] = "2.0" path.write_bytes(preference.sealing.json_bytes(packet)) with self.assertRaisesRegex(preference.sealing.ValidationError, "incompatible"): self.build() path.write_bytes(original) inventory = json.loads((self.f.output/"archive-inventory.json").read_bytes()) (self.f.output/inventory["records"][0]["path"]).write_bytes(b"artificial corruption") with self.assertRaisesRegex(preference.sealing.ValidationError, "copy validation"): self.build() def test_duplicate_forms_records_and_identity_mismatches_rejected(self): key = self.build() rater = next(iter(key["independent_rater_aliases"].values())) path = self.form_path(rater) original = path.read_bytes() with self.assertRaisesRegex(preference.sealing.ValidationError, "duplicate rating series"): self.summarize([path, path]) for change, error in ((lambda f: f["ratings"].append(copy.deepcopy(f["ratings"][0])), "duplicate pair rating"), (lambda f: f["ratings"][0].update(left_sha256="0"*64), "identity or hash mismatch"), (lambda f: f.update(pair_form_version="2.0"), "incompatible"), (lambda f: f["ratings"][0]["clarity"].update(judgment=1), "invalid preference judgment")): form = json.loads(original) change(form) path.write_bytes(preference.sealing.json_bytes(form)) with self.assertRaisesRegex(preference.sealing.ValidationError, error): self.summarize([path]) def test_key_tampering_and_output_destinations_rejected(self): with self.assertRaisesRegex(preference.sealing.ValidationError, "outside sealed archives"): self.build(rater_dir=self.f.output/"new-raters") with self.assertRaisesRegex(preference.sealing.ValidationError, "disjoint"): self.build(rater_dir=self.f.root/"same", coord_dir=self.f.root/"same") key = self.build() with self.assertRaisesRegex(preference.sealing.ValidationError, "existing output"): self.build() key["comparisons"][0]["left_condition_id"] = "bad-condition" self.key_path.write_bytes(preference.sealing.json_bytes(key)) with self.assertRaisesRegex(preference.sealing.ValidationError, "side identity/hash/randomization"): self.summarize() def test_read_only_inputs_and_cli_error_no_success_output(self): def snapshot(root): return {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file()} original = snapshot(self.f.output) key = self.build() coordinator = snapshot(self.coord_dir) rater = snapshot(self.rater_dir) self.summarize() self.assertEqual(original, snapshot(self.f.output)) self.assertEqual(coordinator, snapshot(self.coord_dir)) self.assertEqual(rater, snapshot(self.rater_dir)) stdout, stderr = io.StringIO(), io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = preference.main(["summarize", "--key", str(self.key_path), "--registry", str(self.f.registry_path), "--replicates", "1"]) self.assertEqual(code, 2) self.assertEqual(stdout.getvalue(), "") if __name__ == "__main__": unittest.main()