"""Temporary artificial trajectories, text and binary fixtures; no study findings.""" import contextlib import copy from datetime import datetime, timezone import io import json from pathlib import Path import sys import unittest from unittest import mock import assess_repeated_work as repeated sys.path.insert(0, str(Path(__file__).resolve().parent.parent/"run-sealing")) import test_seal_runs as fixtures class RepeatedWorkTests(unittest.TestCase): def setUp(self): self.f = fixtures.SealingTests() self.f.setUp() self.addCleanup(self.f.doCleanups) f = self.f uniform = "\ufeffArtificial Café output. α.\r\n".encode() for rid in ("fixture-a-other", "fixture-a-endpoint"): (f.root/(rid+".source")).write_bytes(uniform) f.contents[rid] = uniform f.add_record("fixture-a-middle", "artifact", uniform) f.add_record("fixture-prompt-next", "submitted_prompt", b"Artificial worker instruction; intrinsic method label remains.\r\n") f.source["runs"][0]["record_ids"] += ["fixture-a-middle", "fixture-prompt-next"] f.source["runs"][0]["attempts"].append(dict(attempt_id="fixture-request-next", phase="adaptation", role="worker", status="succeeded", retry_of=None, started_at_utc=None, ended_at_utc=None, record_ids=["fixture-prompt-next"])) f.source["records"] = [r for r in f.source["records"] if r["record_id"] != "fixture-usage"] for run in f.source["runs"]: run["record_ids"].remove("fixture-usage") f.source["conditions"].append(dict(condition_id="fixture-second-condition", repetition_ids=["fixture-repetition"])) f.source["runs"][1]["condition_id"] = "fixture-second-condition" f.set_availability() for run in f.source["runs"]: run["record_availability"]["usage_records"] = dict(state="unavailable", note="Artificial usage unavailable") f.seal() commit = json.loads((f.output/"packet-build-manifest.json").read_bytes()) self.manifest = dict(repeated_manifest_version="1.0", assessment_id="artificial-repeated-work-assessment", independent_rater_ids=["source-rater-a", "source-rater-b"], adjudicator_id="source-adjudicator", archives=[dict(archive_id="fixture-archive", path=f.output.name)], runs=[]) for index, source in enumerate(f.source["runs"]): run = {k: source[k] for k in ("run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id")} run.update(archive_id="fixture-archive", seal_record_id=commit["seal_record_id"], trajectory_status="reviewed", trajectory_note="Artificial coordinator note: "+source["condition_id"], declared_delivery_count=3 if index == 0 else 1, deliveries=[], transitions=[]) record_ids = ["fixture-a-other", "fixture-a-middle", "fixture-a-endpoint"] if index == 0 else ["fixture-b-endpoint"] for sequence, rid in enumerate(record_ids, 1): run["deliveries"].append(dict(sequence=sequence, delivered_at_utc=f"2000-01-01T00:00:0{2*(sequence-1)}Z", artifact=dict(record_id=rid, sha256=repeated.sealing.sha(f.contents[rid])))) if index == 0: for seq, prompt, attempt in ((1, "fixture-prompt-a", "fixture-request-a"), (2, "fixture-prompt-next", "fixture-request-next")): run["transitions"].append(dict(transition_id="fixture-transition-"+str(seq), earlier_sequence=seq, later_sequence=seq+1, instruction_status="complete", instruction_note=None, instructions=[dict(record_id=prompt, sha256=repeated.sealing.sha(f.contents[prompt]), attempt_id=attempt, submitted_at_utc=f"2000-01-01T00:00:0{2*seq-1}Z")])) self.manifest["runs"].append(run) self.manifest_path = f.root/"transition-selection.json" self.raters, self.coordinator = f.root/"repetition-raters", f.root/"private-repetition-key" self.key_path = self.coordinator/"repeated-key.json" self.save() def save(self): self.manifest_path.write_bytes(repeated.sealing.json_bytes(self.manifest)) def now(self): return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") def build(self, seed=0, raters=None, coordinator=None): repeated.build(self.manifest_path, raters or self.raters, coordinator or self.coordinator, self.f.registry_path, seed) return json.loads(((coordinator or self.coordinator)/"repeated-key.json").read_bytes()) def form_path(self, alias): return self.raters/alias/"rating-form.json" def evidence(self, alias, artifact): text = (self.raters/alias/"artifacts"/(artifact["artifact_id"]+".txt")).read_bytes().decode("utf-8") return dict(artifact_id=artifact["artifact_id"], sha256=artifact["sha256"], start=0, end=min(12, len(text)), text=text[:12]) def fill(self, key, alias, codes, adjudicate=False): path = self.form_path(alias) form = json.loads(path.read_bytes()) for row, code in zip(form["ratings"], codes): if code is None: continue row.update(repeated_work=code, reason="Artificial binary fixture only, no substantive semantic judgment.", reviewed_scope="Artificial before/after, instructions and public-packet fixture.", completed_at_utc=self.now()) if code == 1: row["completed_work_evidence"] = [self.evidence(alias, row["before"])] row["redone_work_evidence"] = [self.evidence(alias, row["after"])] if adjudicate: basis = {} for rater in key["independent_rater_aliases"].values(): original = json.loads(self.form_path(rater).read_bytes()) basis[rater] = next(r["repeated_work"] for r in original["ratings"] if r["transition_id"] == row["transition_id"]) row.update(independent_judgments=basis, resolution_note="Artificial resolution fixture only.") form["locked_at_utc"] = self.now() path.write_bytes(repeated.sealing.json_bytes(form)) return path def summary(self, paths=()): return repeated.summarize(self.key_path, paths, self.f.registry_path) def test_unchanged_bytes_blank_codes_and_metadata_separation(self): key = self.build() for spec in key["forms"]: root = self.raters/spec["assessor_id"] for run in key["trajectories"]: for artifact in run["artifacts"]: self.assertEqual((root/"artifacts"/(artifact["artifact_id"]+".txt")).read_bytes(), self.f.contents[artifact["source_record_id"]]) for edge in run["transitions"]: for instruction in edge["instruction_sources"]: self.assertEqual((root/"instructions"/(instruction["instruction_id"]+".txt")).read_bytes(), self.f.contents[instruction["record_id"]]) for path in root.rglob("*"): if path.is_file(): content = path.read_bytes() for forbidden in (b"condition_id", b"fixture-condition", b"fixture-run-", b"fixture-request-", b"artificial_configuration"): self.assertNotIn(forbidden, content) self.assertNotIn("fixture-run-", str(path.relative_to(root))) form = json.loads((root/"rating-form.json").read_bytes()) self.assertTrue(all(r["repeated_work"] is None for r in form["ratings"])) index = json.loads((root/"packet-index.json").read_bytes()) for run in index["trajectories"]: self.assertEqual([e["earlier_sequence"] for e in run["transitions"]], sorted(e["earlier_sequence"] for e in run["transitions"])) def test_identical_text_can_receive_zero_and_is_never_auto_coded(self): key = self.build() run = key["trajectories"][0] self.assertEqual(len({a["sha256"] for a in run["artifacts"]}), 1) a = next(iter(key["independent_rater_aliases"].values())) self.assertEqual(self.summary()["independent"][a]["overall"]["missing_judgments"], 2) path = self.fill(key, a, [0, 0]) group = self.summary([path])["independent"][a]["overall"] self.assertEqual(group["repeated_work_rate"], 0) self.assertEqual(group["assessed_transitions"], 2) def test_counts_denominators_affected_runs_and_independent_agreement(self): key = self.build() a, b = key["independent_rater_aliases"].values() p1 = self.fill(key, a, [1, 0]) p2 = self.fill(key, b, [0, 0]) report = self.summary([p1, p2]) group = report["independent"][a]["overall"] self.assertEqual((group["repeated_work_transitions"], group["assessed_transitions"], group["repeated_work_rate"]), (1, 2, .5)) self.assertEqual(group["affected_run_rate"], dict(numerator=1, denominator=2, rate=.5)) self.assertEqual(group["reviewed_no_transition_runs"], 1) self.assertIsNone(report["adjudicated"]["overall"]["repeated_work_rate"]) self.assertEqual(report["independent_rater_agreement"]["overall"]["raw_agreement"], .5) empty = report["independent"][a]["by_condition"]["fixture-second-condition"] self.assertTrue(empty["declared_coverage_complete"]) self.assertIsNone(empty["repeated_work_rate"]) def test_missing_ratings_lower_bound_and_joint_denominator(self): key = self.build() a, b = key["independent_rater_aliases"].values() p1 = self.fill(key, a, [1, None]) p2 = self.fill(key, b, [1, 0]) report = self.summary([p1, p2]) group = report["independent"][a]["overall"] self.assertEqual((group["missing_judgments"], group["missing_transition_coverage"], group["confirmed_affected_runs"]), (1, 1, 1)) self.assertIsNone(group["repeated_work_rate"]) self.assertIsNone(group["affected_run_rate"]["rate"]) self.assertEqual(report["independent_rater_agreement"]["overall"]["jointly_assessed_transition_denominator"], 1) def test_missing_middle_artifact_cannot_be_bridged_or_scored(self): self.manifest["runs"][0]["deliveries"].pop(1) self.save() key = self.build() a = next(iter(key["independent_rater_aliases"].values())) group = self.summary()["independent"][a]["overall"] self.assertEqual((group["expected_known_transitions"], group["missing_transition_inputs"], group["available_transitions"]), (2, 2, 0)) path = self.fill(key, a, [0, None]) with self.assertRaisesRegex(repeated.sealing.ValidationError, "input-incomplete"): self.summary([path]) def test_absent_transition_declaration_is_missing_coverage(self): self.manifest["runs"][0]["transitions"].pop() self.save() key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, [1, None]) group = self.summary([path])["independent"][a]["overall"] self.assertEqual((group["missing_transition_inputs"], group["missing_judgments"], group["missing_transition_coverage"]), (1, 0, 1)) self.assertIsNone(group["repeated_work_rate"]) self.assertEqual(group["confirmed_affected_runs"], 1) def test_partial_instruction_set_keeps_transition_unscored(self): self.manifest["runs"][0]["transitions"][0].update(instruction_status="unfinished", instruction_note="Artificial incomplete instruction set") self.save() key = self.build() a = next(iter(key["independent_rater_aliases"].values())) group = self.summary()["independent"][a]["overall"] self.assertEqual((group["missing_transition_inputs"], group["missing_judgments"]), (1, 1)) path = self.fill(key, a, [0, None]) with self.assertRaisesRegex(repeated.sealing.ValidationError, "input-incomplete"): self.summary([path]) def test_unfinished_and_unavailable_trajectories_do_not_become_empty_runs(self): original = copy.deepcopy(self.manifest) for state in ("unfinished", "unavailable"): self.manifest = copy.deepcopy(original) run = self.manifest["runs"][0] run["trajectory_status"] = state if state == "unavailable": run.update(declared_delivery_count=None, deliveries=[], transitions=[]) self.save() raters, coordinator = self.f.root/(state+"-raters"), self.f.root/(state+"-key") key = self.build(raters=raters, coordinator=coordinator) group = repeated.summarize(coordinator/"repeated-key.json", [], self.f.registry_path)["adjudicated"]["overall"] self.assertFalse(group["declared_coverage_complete"]) self.assertIsNone(group["repeated_work_rate"]) self.assertEqual(group["reviewed_no_transition_runs"], 1) if state == "unavailable": self.assertIsNone(group["runs"][0]["expected_transitions"]) self.assertEqual(group["unavailable_trajectory_runs"], 1) def test_explicit_ordinal_order_accepts_unknown_clocks(self): for run in self.manifest["runs"]: for d in run["deliveries"]: d["delivered_at_utc"] = None for t in run["transitions"]: for instruction in t["instructions"]: instruction["submitted_at_utc"] = None self.save() key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, [0, 0]) self.assertTrue(self.summary([path])["independent"][a]["overall"]["declared_coverage_complete"]) def test_duplicate_nonadjacent_and_reversed_chronology_rejected(self): original = copy.deepcopy(self.manifest) changes = ( (lambda r: r["transitions"].append(copy.deepcopy(r["transitions"][0])), "duplicate transition"), (lambda r: r["transitions"][0].update(later_sequence=3), "reversed/nonadjacent"), (lambda r: r["transitions"][0].update(earlier_sequence=2, later_sequence=1), "reversed/nonadjacent"), (lambda r: r["deliveries"].reverse(), "delivery chronology"), (lambda r: r["deliveries"][1].update(delivered_at_utc="1999-01-01T00:00:00Z"), "delivery chronology"), (lambda r: r["transitions"][0]["instructions"][0].update(submitted_at_utc="1999-01-01T00:00:00Z"), "follow earlier delivery"), (lambda r: r["transitions"][0]["instructions"][0].update(submitted_at_utc="2000-01-01T00:00:05Z"), "precede later delivery"), ) for change, error in changes: self.manifest = copy.deepcopy(original) change(self.manifest["runs"][0]) self.save() with self.assertRaisesRegex(repeated.sealing.ValidationError, error): self.build() self.assertFalse(self.raters.exists()) def test_instruction_membership_hash_identity_and_version_rejected(self): original = copy.deepcopy(self.manifest) cases = ( (lambda r: r["transitions"][0]["instructions"][0].update(attempt_id="fixture-incomplete"), "adaptation-worker attempt"), (lambda r: r["transitions"][0]["instructions"][0].update(sha256="0"*64), "source hash mismatch"), (lambda r: r["deliveries"][0]["artifact"].update(record_id="fixture-b-endpoint"), "record kind/run identity"), (lambda r: r.update(rubric_version="2.0"), "identity/task/version"), ) for change, error in cases: self.manifest = copy.deepcopy(original) change(self.manifest["runs"][0]) self.save() with self.assertRaisesRegex(repeated.sealing.ValidationError, error): self.build() def test_evidence_hash_scope_future_and_locations_rejected(self): key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, [1, 0]) original = path.read_bytes() future = key["trajectories"][0]["artifacts"][1] wrong_run = key["trajectories"][1]["artifacts"][0] other_instruction = key["trajectories"][0]["transitions"][1]["instructions"][0] cases = ( (lambda f: f["ratings"][0]["completed_work_evidence"][0].update(sha256="0"*64), "evidence identity/hash"), (lambda f: f["ratings"][0].update(completed_work_evidence=[self.evidence(a, future)]), "cannot cite future"), (lambda f: f["ratings"][0].update(completed_work_evidence=[self.evidence(a, wrong_run)]), "wrong trajectory"), (lambda f: f["ratings"][0]["redone_work_evidence"][0].update(text="wrong"), "quote does not match"), (lambda f: f["ratings"][0].update(completed_work_evidence=[]), "positive judgment requires"), (lambda f: f["ratings"][0].update(task_reason_evidence=[dict(channel="instruction", document_id=other_instruction["instruction_id"], sha256=other_instruction["sha256"], start=0, end=3, text="Art")]), "not supplied for this transition"), ) for change, error in cases: form = json.loads(original) change(form) path.write_bytes(repeated.sealing.json_bytes(form)) with self.assertRaisesRegex(repeated.sealing.ValidationError, error): self.summary([path]) def test_valid_public_and_instruction_reason_references(self): key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, [0, 0]) form = json.loads(path.read_bytes()) row = form["ratings"][0] instruction = row["instructions"][0] public_text = (self.raters/a/"public"/(row["source_packet_sha256"]+".txt")).read_bytes().decode("utf-8") instruction_text = (self.raters/a/"instructions"/(instruction["instruction_id"]+".txt")).read_bytes().decode("utf-8") row["task_reason_evidence"] = [ dict(channel="public_packet", document_id=row["task_id"], sha256=row["source_packet_sha256"], start=0, end=10, text=public_text[:10]), dict(channel="instruction", document_id=instruction["instruction_id"], sha256=instruction["sha256"], start=0, end=10, text=instruction_text[:10])] path.write_bytes(repeated.sealing.json_bytes(form)) self.assertEqual(self.summary([path])["independent"][a]["overall"]["repeated_work_rate"], 0) def test_adjudication_requires_original_locked_judgments_and_stays_separate(self): key = self.build() a, b = key["independent_rater_aliases"].values() p1 = self.fill(key, a, [1, 0]) p2 = self.fill(key, b, [0, 0]) pa = self.fill(key, key["adjudicator_alias"], [0, 0], adjudicate=True) with self.assertRaisesRegex(repeated.sealing.ValidationError, "both independent"): self.summary([pa, p1]) report = self.summary([pa, p1, p2]) self.assertEqual(report["adjudicated"]["overall"]["repeated_work_rate"], 0) self.assertEqual(report["independent"][a]["overall"]["repeated_work_rate"], .5) form = json.loads(pa.read_bytes()) form["ratings"][0]["independent_judgments"][a] = 0 pa.write_bytes(repeated.sealing.json_bytes(form)) with self.assertRaisesRegex(repeated.sealing.ValidationError, "stale adjudication"): self.summary([p1, p2, pa]) def test_duplicate_rating_identity_binary_and_lock_rejections(self): key = self.build() a = next(iter(key["independent_rater_aliases"].values())) path = self.fill(key, a, [0, 0]) original = path.read_bytes() with self.assertRaisesRegex(repeated.sealing.ValidationError, "duplicate assessment series"): self.summary([path, path]) for change, error in ( (lambda f: f["ratings"].append(copy.deepcopy(f["ratings"][0])), "duplicate assessed transition"), (lambda f: f["ratings"][0]["before"].update(sha256="0"*64), "identity or hash mismatch"), (lambda f: f["ratings"][0].update(repeated_work=True), "expected integer"), (lambda f: f["ratings"][0].update(condition_id="excluded"), "unexpected/missing fields"), (lambda f: f.update(repeated_form_version="2.0"), "incompatible"), (lambda f: f.update(locked_at_utc=None), "precede form lock"), ): form = json.loads(original) change(form) path.write_bytes(repeated.sealing.json_bytes(form)) with self.assertRaisesRegex(repeated.sealing.ValidationError, error): self.summary([path]) def test_unsealed_and_key_tampering_rejected(self): commit_path = self.f.output/"packet-build-manifest.json" content = commit_path.read_bytes() commit_path.unlink() with self.assertRaisesRegex(repeated.sealing.ValidationError, "unsealed"): self.build() commit_path.write_bytes(content) key = self.build() key["trajectories"][0]["condition_id"] = "wrong-condition" self.key_path.write_bytes(repeated.sealing.json_bytes(key)) with self.assertRaisesRegex(repeated.sealing.ValidationError, "key source identity/hash"): self.summary() def test_read_only_inputs_overwrite_refusal_and_cli(self): def snapshot(root): return {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file()} archive = snapshot(self.f.output) with self.assertRaisesRegex(repeated.sealing.ValidationError, "outside sealed archives"): self.build(raters=self.f.output/"new-raters") with self.assertRaisesRegex(repeated.sealing.ValidationError, "disjoint"): self.build(raters=self.f.root/"same", coordinator=self.f.root/"same") self.build() raters, coordinator = snapshot(self.raters), snapshot(self.coordinator) with self.assertRaisesRegex(repeated.sealing.ValidationError, "existing output"): self.build() self.manifest_path.unlink() stdout, stderr = io.StringIO(), io.StringIO() with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = repeated.main(["summarize", "--key", str(self.key_path), "--registry", str(self.f.registry_path)]) self.assertEqual(code, 0) self.assertEqual(json.loads(stdout.getvalue())["repeated_report_version"], "1.0") self.assertEqual(archive, snapshot(self.f.output)) self.assertEqual(raters, snapshot(self.raters)) self.assertEqual(coordinator, snapshot(self.coordinator)) def test_failed_writes_no_success_record_and_duplicate_json_rejected(self): stdout, stderr = io.StringIO(), io.StringIO() with mock.patch.object(repeated.packets, "write_bytes", side_effect=OSError("artificial write failure")): with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = repeated.main(["build", "--manifest", str(self.manifest_path), "--rater-output", str(self.raters), "--coordinator-output", str(self.coordinator), "--registry", str(self.f.registry_path)]) self.assertEqual(code, 2) self.assertFalse(self.key_path.exists()) self.assertEqual(stdout.getvalue(), "") with self.assertRaisesRegex(repeated.sealing.ValidationError, "duplicate JSON key"): repeated.sealing.parse_json(b'{"repeated_work":0,"repeated_work":1}', "artificial duplicate") if __name__ == "__main__": unittest.main()