"""Temporary artificial metadata/text and numeric ratings; no study observations.""" import contextlib import copy from datetime import datetime, timezone import hashlib import io import json from pathlib import Path import sys import unittest from unittest import mock import analyze_progress as progress sys.path.insert(0, str(Path(__file__).resolve().parent.parent/"run-sealing")) import test_seal_runs as archival_fixtures import test_score_forms as numeric_fixtures class ProgressTests(unittest.TestCase): def setUp(self): self.archive = archival_fixtures.SealingTests() self.archive.setUp() self.addCleanup(self.archive.doCleanups) f = self.archive self.condition, self.repetitions = "numeric-fixture-condition", ["fixture-repeat-1", "fixture-repeat-2"] f.source["conditions"] = [dict(condition_id=self.condition, repetition_ids=self.repetitions), dict(condition_id="absent-fixture-condition", repetition_ids=[self.repetitions[0]])] f.source["records"] = [r for r in f.source["records"] if r["record_id"] in ("fixture-config", "fixture-prompt-a")] f.source["runs"] = [] self.artifacts = {} for task in f.registry["tasks"]: for rep in self.repetitions: rid = "artificial-run-"+task["task_id"]+"-"+rep records = [] for turn in (1, 2, 3): aid = rid+"-turn-"+str(turn) content = ("Artificial UTF-8 snapshot. Café.\r\n"+aid+"\r\n").encode() f.add_record(aid, "artifact", content) self.artifacts[(task["task_id"], rep, turn)] = dict(record_id=aid, sha256=progress.sealing.sha(content)) records.append(aid) f.source["runs"].append(dict(run_id=rid, condition_id=self.condition, task_id=task["task_id"], task_version="1.0", rubric_version="1.0", repetition_id=rep, run_status="completed", ended_at_utc=None, endpoint_selection=dict(artifact_id=records[-1], basis="Explicit artificial final endpoint; no best-score choice"), record_ids=["fixture-config", "fixture-prompt-a", *records], record_availability={}, attempts=[dict(attempt_id="attempt-"+rid, phase="adaptation", role="worker", status="succeeded", retry_of=None, started_at_utc=None, ended_at_utc=None, record_ids=["fixture-prompt-a"])])) f.set_availability() f.seal() commit = json.loads((f.output/"packet-build-manifest.json").read_bytes()) self.fixtures = {} for turn in (1, 2, 3): fixture = numeric_fixtures.ScoringTests() fixture.setUp() self.addCleanup(fixture.doCleanups) fixture.add_repetition(self.repetitions[1]) fixture.manifest["batch_id"] = f"turn-{turn}-"+fixture.manifest["batch_id"] fixture.manifest["seal_record_id"] = f"turn-{turn}-"+fixture.manifest["seal_record_id"] for assignment in fixture.manifest["assignments"]: assignment["blinded_artifact_id"] = f"turn-{turn}-"+assignment["blinded_artifact_id"] assignment["blinded_artifact_sha256"] = self.artifacts[(assignment["task_id"], assignment["repetition_id"], turn)]["sha256"] for index, form in enumerate(fixture.forms): form.update(form_id=f"turn-{turn}-"+form["form_id"], batch_id=fixture.manifest["batch_id"], seal_record_id=fixture.manifest["seal_record_id"]) count = ((0, 6, 3) if index == 0 else (12, 6, 12))[turn-1] for row in form["ratings"]: row["rating_record_id"] = f"turn-{turn}-"+row["rating_record_id"] row["blinded_artifact_id"] = f"turn-{turn}-"+row["blinded_artifact_id"] assignment = next(a for a in fixture.manifest["assignments"] if a["blinded_artifact_id"] == row["blinded_artifact_id"]) row["blinded_artifact_sha256"] = assignment["blinded_artifact_sha256"] fixture.set_values(row, [1]*count+[0]*(12-count)) row["completed_at_utc"] = self.now() form["locked_at_utc"] = self.now() self.fixtures[turn] = fixture self.manifest = dict(progress_manifest_version="1.0", analysis_id="artificial-progress-analysis", turn_count=3, independent_rater_ids=["analysis-rater-a", "analysis-rater-b"], conditions=[dict(condition_id=self.condition, repetition_ids=self.repetitions)], archives=[dict(archive_id="artificial-archive", path=str(f.output))], assessment_sets=[], runs=[]) for source in f.source["runs"]: run = {k: source[k] for k in ("run_id", "condition_id", "task_id", "task_version", "rubric_version", "repetition_id")} run.update(archive_id="artificial-archive", seal_record_id=commit["seal_record_id"], snapshots=[]) for turn, fixture in self.fixtures.items(): assignment = next(a for a in fixture.manifest["assignments"] if a["task_id"] == run["task_id"] and a["repetition_id"] == run["repetition_id"]) run["snapshots"].append(dict(turn=turn, delivered_at_utc=f"2000-01-01T00:00:0{turn}Z", artifact=copy.deepcopy(self.artifacts[(run["task_id"], run["repetition_id"], turn)]), assessment=dict(assessment_set_id="assessment-turn-"+str(turn), blinded_artifact_id=assignment["blinded_artifact_id"]))) self.manifest["runs"].append(run) self.manifest_path = f.root/"progress-selection.json" self.output = f.root/"new-progress-report.json" def now(self): return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") def descriptor(self, path): return dict(path=str(path), sha256=hashlib.sha256(path.read_bytes()).hexdigest()) def prepare(self, adjudicated=True): sets = [] for turn, fixture in self.fixtures.items(): coordinator = fixture.write("coordinator.json", fixture.manifest) ratings = [fixture.write(f"ratings-{i}.json", form) for i, form in enumerate(fixture.forms)] adj_paths = [] if adjudicated: form = fixture.adjudication() for row in form["adjudications"]: row["completed_at_utc"] = self.now() form["locked_at_utc"] = self.now() adj_paths = [fixture.write("adjudication.json", form)] sets.append(dict(assessment_set_id="assessment-turn-"+str(turn), coordinator_manifest=self.descriptor(coordinator), rating_forms=[self.descriptor(p) for p in ratings], adjudication_forms=[self.descriptor(p) for p in adj_paths], rater_mapping=dict(zip(self.manifest["independent_rater_ids"], fixture.raters)))) self.manifest["assessment_sets"] = sets self.save_manifest() def save_manifest(self): self.manifest_path.write_bytes(progress.sealing.json_bytes(self.manifest)) def analyze(self, adjudicated=True): self.prepare(adjudicated) return progress.analyze_progress(self.manifest_path, self.archive.registry_path) def condition_result(self, report, rater="analysis-rater-a"): return report["independent"][rater]["by_condition"][self.condition] def test_normalized_scores_signed_changes_and_series_separation(self): report = self.analyze() result = self.condition_result(report) self.assertEqual(result["fully_observed_score_curve"], [0, .5, .25]) self.assertEqual(result["fully_observed_change_curve"], [.5, -.25]) self.assertIsNone(result["turns"][0]["signed_adjacent_change"]["mean"]) self.assertEqual(result["turns"][0]["signed_adjacent_change"]["expected_cells"], 0) self.assertEqual(self.condition_result(report, "analysis-rater-b")["fully_observed_score_curve"], [1, .5, 1]) self.assertEqual(report["adjudicated"]["by_condition"][self.condition]["fully_observed_score_curve"], [1, .5, 1]) def test_equal_repetition_task_and_family_weighting(self): fixture = self.fixtures[2] first_family = self.archive.registry["families"][0]["task_ids"] for assignment, row in zip(fixture.manifest["assignments"], fixture.forms[0]["ratings"]): if assignment["task_id"] not in first_family: count = 12 else: count = 12 if assignment["task_id"] == first_family[0] and assignment["repetition_id"] == self.repetitions[0] else 0 fixture.set_values(row, [1]*count+[0]*(12-count)) point = self.condition_result(self.analyze())["turns"][1]["level"] self.assertEqual(point["task_means"][first_family[0]], .5) family = self.archive.registry["families"][0]["family_id"] self.assertEqual(point["family_means"][family], .125) self.assertAlmostEqual(point["mean"], (.125+1+1)/3) def test_missing_turns_never_bridge_or_carry_forward(self): for run in self.manifest["runs"]: run["snapshots"].pop(1) report = self.analyze() result = self.condition_result(report) self.assertEqual([p["level"]["mean"] for p in result["turns"]], [0, None, .25]) self.assertEqual([p["signed_adjacent_change"]["mean"] for p in result["turns"]], [None, None, None]) self.assertEqual(result["turns"][1]["coverage"]["missing_snapshots"], 24) self.assertIsNone(result["fully_observed_score_curve"]) self.assertIsNone(result["fully_observed_change_curve"]) self.assertEqual(len(report["unused_assessment_assignments"]), 24) def test_partial_criteria_and_resolution_status_withhold_score(self): fixture = self.fixtures[2] fixture.set_values(fixture.forms[0]["ratings"][0], [None]+[1]*11) report = self.analyze() point = self.condition_result(report)["turns"][1] self.assertEqual(point["coverage"]["ineligible_records"], 1) self.assertEqual(point["level"]["assessed_cells"], 23) self.assertIsNone(point["level"]["mean"]) self.assertIsNotNone(report["adjudicated"]["by_condition"][self.condition]["turns"][1]["level"]["mean"]) fixture.set_values(fixture.forms[0]["ratings"][0], [1]*12) fixture.forms[0]["ratings"][0]["assessment_status"] = "needs_resolution" point = self.condition_result(self.analyze())["turns"][1] self.assertIsNone(point["level"]["mean"]) self.assertEqual(point["coverage"]["observed_criterion_judgments"], 24*12) def test_missing_record_and_assignment_are_distinct(self): self.fixtures[2].forms[0]["ratings"].pop(0) self.manifest["runs"][1]["snapshots"][1]["assessment"] = None point = self.condition_result(self.analyze(adjudicated=False))["turns"][1] self.assertEqual(point["coverage"]["missing_assessment_records"], 1) self.assertEqual(point["coverage"]["missing_assessment_assignments"], 1) self.assertEqual(point["level"]["missing_cells"], 2) def test_missing_run_and_declared_condition_are_not_available_case_means(self): self.manifest["runs"].pop() self.manifest["conditions"].append(dict(condition_id="absent-fixture-condition", repetition_ids=[self.repetitions[0]])) report = self.analyze() result = self.condition_result(report) self.assertEqual(result["turns"][0]["coverage"]["missing_run_cells"], 1) self.assertIsNone(result["turns"][0]["level"]["mean"]) missing = report["independent"]["analysis-rater-a"]["by_condition"]["absent-fixture-condition"] self.assertEqual(missing["turns"][0]["level"]["expected_cells"], 12) self.assertEqual(missing["turns"][0]["level"]["assessed_cells"], 0) self.assertIsNone(missing["fully_observed_score_curve"]) def test_prescribed_endpoint_retained_even_when_earlier_score_is_higher(self): result = self.condition_result(self.analyze()) row = result["runs"][0] self.assertGreater(row["turns"][1]["normalized_requirement_satisfaction"], row["turns"][2]["normalized_requirement_satisfaction"]) self.assertEqual(row["prescribed_endpoint"]["artifact_record_id"], row["turns"][2]["artifact"]["record_id"]) self.assertEqual(row["prescribed_endpoint"]["listed_turn"], 3) self.assertEqual(row["prescribed_endpoint"]["sha256"], row["turns"][2]["artifact"]["sha256"]) self.assertFalse(row["prescribed_endpoint"]["endpoint_changed"]) def test_single_turn_has_no_change_baseline_or_final_endpoint_substitution(self): self.manifest["turn_count"] = 1 for run in self.manifest["runs"]: run["snapshots"] = run["snapshots"][:1] result = self.condition_result(self.analyze()) self.assertTrue(result["curve_complete"]) self.assertIsNone(result["adjacent_change_curve_complete"]) self.assertIsNone(result["fully_observed_change_curve"]) self.assertIsNone(result["turns"][0]["signed_adjacent_change"]["mean"]) self.assertIsNone(result["runs"][0]["prescribed_endpoint"]["listed_turn"]) def test_empty_assessments_return_missing_not_fabricated_scores(self): self.prepare() for source in self.manifest["assessment_sets"]: source.update(rating_forms=[], adjudication_forms=[]) self.save_manifest() report = progress.analyze_progress(self.manifest_path, self.archive.registry_path) result = self.condition_result(report) self.assertIsNone(result["fully_observed_score_curve"]) self.assertEqual(result["turns"][0]["coverage"]["missing_assessment_records"], 24) def test_duplicate_turns_reversed_order_and_clocks_rejected(self): original = copy.deepcopy(self.manifest["runs"][0]["snapshots"]) for change in (lambda s: s.append(copy.deepcopy(s[0])), lambda s: s.reverse(), lambda s: s[1].update(delivered_at_utc="1999-01-01T00:00:00Z")): self.manifest["runs"][0]["snapshots"] = copy.deepcopy(original) change(self.manifest["runs"][0]["snapshots"]) with self.assertRaisesRegex(progress.sealing.ValidationError, "delivery chronology"): self.analyze() def test_source_hash_and_assessment_binding_mismatches_rejected(self): self.prepare() self.manifest["assessment_sets"][0]["rating_forms"][0]["sha256"] = "0"*64 self.save_manifest() with self.assertRaisesRegex(progress.sealing.ValidationError, "source hash mismatch"): progress.analyze_progress(self.manifest_path, self.archive.registry_path) self.prepare() self.manifest["runs"][0]["snapshots"][0]["assessment"]["blinded_artifact_id"] = self.manifest["runs"][1]["snapshots"][0]["assessment"]["blinded_artifact_id"] self.save_manifest() with self.assertRaisesRegex(progress.sealing.ValidationError, "identity or hash mismatch"): progress.analyze_progress(self.manifest_path, self.archive.registry_path) def test_no_reuse_of_a_bound_assessment_across_turns(self): self.manifest["runs"][0]["snapshots"][1]["assessment"] = copy.deepcopy(self.manifest["runs"][0]["snapshots"][0]["assessment"]) with self.assertRaisesRegex(progress.sealing.ValidationError, "no carry-forward"): self.analyze() def test_incompatible_rubric_and_invalid_binary_rejected(self): self.manifest["runs"][0]["rubric_version"] = "2.0" with self.assertRaisesRegex(progress.sealing.ValidationError, "identity/task/version mismatch"): self.analyze() self.manifest["runs"][0]["rubric_version"] = "1.0" self.fixtures[1].forms[0]["ratings"][0]["criteria"][0]["satisfaction"] = True with self.assertRaisesRegex(progress.sealing.ValidationError, "expected integer"): self.analyze() def test_rater_mapping_and_unsealed_inputs_rejected(self): self.prepare() mapping = self.manifest["assessment_sets"][0]["rater_mapping"] mapping["analysis-rater-a"] = mapping["analysis-rater-b"] self.save_manifest() with self.assertRaisesRegex(progress.sealing.ValidationError, "must not pool"): progress.analyze_progress(self.manifest_path, self.archive.registry_path) self.prepare() (self.archive.output/"packet-build-manifest.json").unlink() with self.assertRaisesRegex(progress.sealing.ValidationError, "unsealed"): progress.analyze_progress(self.manifest_path, self.archive.registry_path) def test_preseal_assessments_are_rejected(self): self.fixtures[1].forms[0]["ratings"][0]["completed_at_utc"] = "2000-01-01T00:00:00Z" with self.assertRaisesRegex(progress.sealing.ValidationError, "predates corresponding archive"): self.analyze(adjudicated=False) def test_future_assessment_locks_and_duplicate_source_batches_rejected(self): original = self.fixtures[1].forms[0]["locked_at_utc"] self.fixtures[1].forms[0]["locked_at_utc"] = "2999-01-01T00:00:00Z" with self.assertRaisesRegex(progress.sealing.ValidationError, "future assessment lock"): self.analyze(adjudicated=False) self.fixtures[1].forms[0]["locked_at_utc"] = original self.prepare() duplicate = copy.deepcopy(self.manifest["assessment_sets"][0]) duplicate["assessment_set_id"] = "different-alias-same-batch" self.manifest["assessment_sets"].append(duplicate) self.save_manifest() with self.assertRaisesRegex(progress.sealing.ValidationError, "duplicate source assessment batch"): progress.analyze_progress(self.manifest_path, self.archive.registry_path) def test_new_output_and_read_only_inputs(self): self.prepare() input_paths = [self.manifest_path] input_paths += [p for p in self.archive.output.rglob("*") if p.is_file()] input_paths += [p for fixture in self.fixtures.values() for p in fixture.directory.glob("*.json")] before = {str(p): p.read_bytes() for p in input_paths} result = progress.write_report(self.manifest_path, self.output, self.archive.registry_path) self.assertEqual(result["sha256"], hashlib.sha256(self.output.read_bytes()).hexdigest()) self.assertTrue(json.loads(self.output.read_bytes())["coordinator_only"]) self.assertEqual(before, {str(p): p.read_bytes() for p in input_paths}) with self.assertRaisesRegex(progress.sealing.ValidationError, "existing output"): progress.write_report(self.manifest_path, self.output, self.archive.registry_path) with self.assertRaisesRegex(progress.sealing.ValidationError, "outside sealed archives"): progress.write_report(self.manifest_path, self.archive.output/"forbidden-report.json", self.archive.registry_path) def test_failed_publication_no_output_or_success_and_blank_stdout(self): self.prepare() stdout, stderr = io.StringIO(), io.StringIO() with mock.patch.object(progress.os, "link", side_effect=OSError("artificial publication failure")): with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): code = progress.main(["--manifest", str(self.manifest_path), "--registry", str(self.archive.registry_path), "--output", str(self.output)]) self.assertEqual(code, 2) self.assertFalse(self.output.exists()) self.assertEqual(stdout.getvalue(), "") self.assertFalse(list(self.output.parent.glob(".pending-progress-*"))) if __name__ == "__main__": unittest.main()