test_data_integrity.py
python
sha256:91e875d4a97bb1e35f37992f803988d5713931f1782d870c371c50054574af22
Add fixture provenance retention deletion
Human
minor
⚠ breaking
42 days ago
| 1 | """Data-integrity tier tests for provenance and deletion invariants.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import json |
| 6 | import tempfile |
| 7 | import unittest |
| 8 | from pathlib import Path |
| 9 | |
| 10 | from scooling_lab_helpers import PROJECT_ROOT, valid_payload |
| 11 | |
| 12 | from scooling_lab.fake_worker import fixture_dataset_hash |
| 13 | from scooling_lab.service import TrainingApiService |
| 14 | from scooling_lab.store import TrainingJobStore |
| 15 | |
| 16 | |
| 17 | EXPECTED_UNSLOTH_NOTICE = ( |
| 18 | "Files under unsloth/*, tests/*, scripts/* are Apache 2.0 licensed.\n" |
| 19 | "Files under studio/*, unsloth_cli/* which is optional to install are AGPLv3 licensed." |
| 20 | ) |
| 21 | |
| 22 | |
| 23 | class ScoolingLabDataIntegrityTests(unittest.TestCase): |
| 24 | """Data-integrity tests for stable hashes, reloads, and tombstones.""" |
| 25 | |
| 26 | def test_data_integrity_provenance_hashes_stable_across_restarts(self) -> None: |
| 27 | """Persisted jobs reload with the same provenance and artifact hashes.""" |
| 28 | |
| 29 | with tempfile.TemporaryDirectory() as directory: |
| 30 | store_path = Path(directory) / "store.json" |
| 31 | service = TrainingApiService(TrainingJobStore(persistence_path=store_path)) |
| 32 | created = service.create_training_job(valid_payload("restart-stability")) |
| 33 | job_id = str(created["id"]) |
| 34 | first_artifact = service.list_artifacts(job_id)["artifacts"][0] |
| 35 | first_provenance = service.get_provenance(job_id) |
| 36 | |
| 37 | reloaded = TrainingApiService(TrainingJobStore(persistence_path=store_path)) |
| 38 | replayed = reloaded.create_training_job(valid_payload("restart-stability")) |
| 39 | second_artifact = reloaded.list_artifacts(job_id)["artifacts"][0] |
| 40 | second_provenance = reloaded.get_provenance(job_id) |
| 41 | |
| 42 | self.assertEqual(replayed["id"], job_id) |
| 43 | self.assertEqual(first_artifact["artifactHash"], second_artifact["artifactHash"]) |
| 44 | self.assertEqual(first_artifact["datasetHash"], second_artifact["datasetHash"]) |
| 45 | self.assertEqual(first_provenance, second_provenance) |
| 46 | |
| 47 | def test_data_integrity_deletion_verification_finds_zero_residue(self) -> None: |
| 48 | """Deletion verification proves deleted hashes are absent from store output.""" |
| 49 | |
| 50 | service = TrainingApiService(TrainingJobStore()) |
| 51 | created = service.create_training_job(valid_payload("zero-residue")) |
| 52 | job_id = str(created["id"]) |
| 53 | artifact = service.list_artifacts(job_id)["artifacts"][0] |
| 54 | provenance = service.get_provenance(job_id) |
| 55 | deleted_hashes = ( |
| 56 | str(artifact["datasetHash"]), |
| 57 | str(artifact["artifactHash"]), |
| 58 | str(provenance["trainingConfigHash"]), |
| 59 | ) |
| 60 | |
| 61 | service.delete_artifact(job_id, str(artifact["id"])) |
| 62 | |
| 63 | self.assertTrue(service.verify_deleted_artifact_absence(deleted_hashes)) |
| 64 | |
| 65 | def test_data_integrity_tombstone_carries_no_content_fields(self) -> None: |
| 66 | """Deleted job tombstones do not carry request, model, dataset, or hash fields.""" |
| 67 | |
| 68 | service = TrainingApiService(TrainingJobStore()) |
| 69 | created = service.create_training_job(valid_payload("tombstone")) |
| 70 | job_id = str(created["id"]) |
| 71 | artifact_id = str(service.list_artifacts(job_id)["artifacts"][0]["id"]) |
| 72 | |
| 73 | service.delete_artifact(job_id, artifact_id) |
| 74 | tombstone_json = json.dumps(service.get_training_job(job_id), sort_keys=True) |
| 75 | |
| 76 | forbidden_terms = ( |
| 77 | "artifactHash", |
| 78 | "datasetHash", |
| 79 | "fixture-tiny-llm", |
| 80 | "fixture:synthetic-tiny-v1", |
| 81 | "request", |
| 82 | "trainingParameters", |
| 83 | ) |
| 84 | for forbidden in forbidden_terms: |
| 85 | with self.subTest(forbidden=forbidden): |
| 86 | self.assertNotIn(forbidden, tombstone_json) |
| 87 | |
| 88 | def test_data_integrity_job_and_artifact_hashes_survive_restart(self) -> None: |
| 89 | """Job id, dataset hash, and artifact hash are stable after reload.""" |
| 90 | |
| 91 | with tempfile.TemporaryDirectory() as directory: |
| 92 | persistence_path = Path(directory) / "jobs.json" |
| 93 | service = TrainingApiService(TrainingJobStore(persistence_path)) |
| 94 | created = service.create_training_job(valid_payload("restart")) |
| 95 | job_id = str(created["id"]) |
| 96 | artifact = service.list_artifacts(job_id)["artifacts"][0] |
| 97 | artifact_hash = str(artifact["artifactHash"]) |
| 98 | dataset_hash = str(artifact["datasetHash"]) |
| 99 | |
| 100 | reloaded = TrainingApiService(TrainingJobStore(persistence_path)) |
| 101 | recreated = reloaded.create_training_job(valid_payload("restart")) |
| 102 | reloaded_artifact = reloaded.list_artifacts(job_id)["artifacts"][0] |
| 103 | |
| 104 | self.assertEqual(recreated["id"], job_id) |
| 105 | self.assertEqual(reloaded_artifact["datasetHash"], dataset_hash) |
| 106 | self.assertEqual(reloaded_artifact["artifactHash"], artifact_hash) |
| 107 | self.assertEqual(dataset_hash, fixture_dataset_hash()) |
| 108 | |
| 109 | def test_data_integrity_unsloth_evidence_preserved_byte_for_byte(self) -> None: |
| 110 | """CI-visible license evidence preserves the pinned candidate and notice.""" |
| 111 | |
| 112 | evidence = (PROJECT_ROOT / "docs/UNSLOTH-LICENSE-EVIDENCE.md").read_text( |
| 113 | encoding="utf-8" |
| 114 | ) |
| 115 | self.assertIn("- Candidate version: `2026.6.1`.", evidence) |
| 116 | self.assertIn(EXPECTED_UNSLOTH_NOTICE, evidence) |
| 117 | |
| 118 | |
| 119 | if __name__ == "__main__": |
| 120 | unittest.main() |
File History
1 commit
sha256:91e875d4a97bb1e35f37992f803988d5713931f1782d870c371c50054574af22
Add fixture provenance retention deletion
Human
minor
⚠
42 days ago