aaronrene / scooling-lab public
test_t3_integration.py python
202 lines 8.8 KB
Raw
sha256:fc4c9ad652d1fff3dc508cb6ea02ee710ee6dfc4cb3761291d9900b5e029ea8a feat(slice-7): T3 dataset review lifecycle, job queue, prov… Human minor ⚠ breaking 43 days ago
1 """Integration tier tests — T3 dataset review + job lifecycle API contract."""
2
3 from __future__ import annotations
4
5 import unittest
6 from datetime import UTC, datetime, timedelta
7
8 from scooling_lab_helpers import valid_payload
9
10 from scooling_lab.dataset_review import DatasetStatus, DatasetStore, RejectionReasonCode
11 from scooling_lab.errors import ApiError, ErrorCode
12 from scooling_lab.provenance import validate_provenance_record
13 from scooling_lab.service import TrainingApiService
14 from scooling_lab.store import TrainingJobStore
15
16
17 def _make_service(
18 *,
19 queue_limit: int = 10,
20 max_concurrent_running: int = 1,
21 dataset_store: DatasetStore | None = None,
22 auto_run: bool = True,
23 ) -> TrainingApiService:
24 """Convenience factory for integration tests."""
25
26 store = TrainingJobStore(
27 queue_limit=queue_limit, max_concurrent_running=max_concurrent_running
28 )
29 return TrainingApiService(
30 store, auto_run_worker=auto_run, dataset_store=dataset_store
31 )
32
33
34 class T3IntegrationDatasetReviewLifecycleTests(unittest.TestCase):
35 """Integration tests for the full dataset registration → review flow."""
36
37 def test_integration_t3_register_review_approve_then_submit_job(self) -> None:
38 """A dataset that goes through the full approval flow allows job creation."""
39
40 ds_store = DatasetStore()
41 ds_store.register("custom-ds-v1")
42 ds_store.submit_for_review("custom-ds-v1")
43 ds_store.approve("custom-ds-v1")
44 # Use a service that also approves the fixture dataset (default).
45 service = TrainingApiService(TrainingJobStore(), dataset_store=ds_store)
46
47 # Standard fixture job still works because fixture is pre-approved.
48 created = service.create_training_job(valid_payload("integration-approved"))
49 self.assertEqual(created["status"], "succeeded")
50
51 def test_integration_t3_unapproved_dataset_blocks_job_at_api_boundary(self) -> None:
52 """A dataset that is only registered (not approved) blocks job submission."""
53
54 ds_store = DatasetStore()
55 ds_store.register("custom-ds-v2")
56 service = TrainingApiService(TrainingJobStore(), dataset_store=ds_store)
57 with self.assertRaises(ApiError) as raised:
58 service.create_training_job(
59 {
60 "idempotencyKey": "integration-blocked",
61 "datasetId": "custom-ds-v2",
62 "modelId": "fixture-tiny-llm",
63 "requestedBy": "integration-test",
64 }
65 )
66 self.assertEqual(raised.exception.code, ErrorCode.DATASET_NOT_APPROVED)
67
68 def test_integration_t3_rejected_dataset_blocks_job(self) -> None:
69 """A rejected dataset is refused at job creation with DATASET_NOT_APPROVED."""
70
71 ds_store = DatasetStore()
72 ds_store.register("custom-ds-v3")
73 ds_store.submit_for_review("custom-ds-v3")
74 ds_store.reject("custom-ds-v3", RejectionReasonCode.POLICY_VIOLATION)
75 service = TrainingApiService(TrainingJobStore(), dataset_store=ds_store)
76 with self.assertRaises(ApiError) as raised:
77 service.create_training_job(
78 {
79 "idempotencyKey": "integration-rejected",
80 "datasetId": "custom-ds-v3",
81 "modelId": "fixture-tiny-llm",
82 "requestedBy": "integration-test",
83 }
84 )
85 self.assertEqual(raised.exception.code, ErrorCode.DATASET_NOT_APPROVED)
86
87 def test_integration_t3_rejection_reason_in_dataset_record(self) -> None:
88 """Rejection reason code is visible in the dataset record's public dict."""
89
90 ds_store = DatasetStore()
91 ds_store.register("reason-ds-v1")
92 ds_store.submit_for_review("reason-ds-v1")
93 ds_store.reject("reason-ds-v1", RejectionReasonCode.SCHEMA_MISMATCH)
94 record = ds_store.get("reason-ds-v1")
95 public = record.to_public_dict()
96 self.assertEqual(public["rejectionReasonCode"], "SCHEMA_MISMATCH")
97 self.assertEqual(public["status"], DatasetStatus.REJECTED.value)
98
99 def test_integration_t3_dataset_status_does_not_expose_free_text(self) -> None:
100 """Public dataset record contains no free-text fields from request payload."""
101
102 ds_store = DatasetStore()
103 ds_store.register("clean-ds-v1")
104 ds_store.submit_for_review("clean-ds-v1")
105 ds_store.reject("clean-ds-v1", RejectionReasonCode.FORMAT_INVALID)
106 public_str = str(ds_store.get("clean-ds-v1").to_public_dict())
107 self.assertNotIn("caller message", public_str)
108 self.assertIn("FORMAT_INVALID", public_str)
109
110
111 class T3IntegrationQueueStateTests(unittest.TestCase):
112 """Integration tests for queue state inspection."""
113
114 def test_integration_t3_queue_state_reflects_active_jobs(self) -> None:
115 """Queue state accurately counts queued/running/active jobs."""
116
117 service = _make_service(auto_run=False, max_concurrent_running=2, queue_limit=20)
118 initial = service.get_queue_state()
119 self.assertEqual(initial["queuedCount"], 0)
120 self.assertEqual(initial["runningCount"], 0)
121 self.assertEqual(initial["activeCount"], 0)
122
123 service.create_training_job(valid_payload("queue-state-a"))
124 service.create_training_job(valid_payload("queue-state-b"))
125 after = service.get_queue_state()
126 self.assertEqual(after["queuedCount"], 2)
127 self.assertEqual(after["maxConcurrentRunning"], 2)
128
129 def test_integration_t3_queue_state_after_completion(self) -> None:
130 """Completed jobs do not count in the active queue."""
131
132 service = _make_service(auto_run=True, max_concurrent_running=5, queue_limit=20)
133 service.create_training_job(valid_payload("complete-queue"))
134 after = service.get_queue_state()
135 self.assertEqual(after["queuedCount"], 0)
136 self.assertEqual(after["runningCount"], 0)
137 self.assertEqual(after["activeCount"], 0)
138
139
140 class T3IntegrationProvenanceOnCompletionTests(unittest.TestCase):
141 """Integration tests confirming provenance is always emitted via the Slice-5 validator."""
142
143 def test_integration_t3_every_succeeded_job_has_valid_provenance(self) -> None:
144 """Succeeded jobs always expose a schema-valid provenance record."""
145
146 service = _make_service()
147 for suffix in ("prov-a", "prov-b", "prov-c"):
148 created = service.create_training_job(valid_payload(suffix))
149 job_id = str(created["id"])
150 self.assertEqual(created["status"], "succeeded")
151 prov = service.get_provenance(job_id)
152 validate_provenance_record(prov)
153 self.assertEqual(prov["jobId"], job_id)
154
155 def test_integration_t3_provenance_links_to_artifact(self) -> None:
156 """Provenance artifact hash matches the artifact in the job's artifact list."""
157
158 service = _make_service()
159 created = service.create_training_job(valid_payload("prov-link"))
160 job_id = str(created["id"])
161 artifact = service.list_artifacts(job_id)["artifacts"][0]
162 prov = service.get_provenance(job_id)
163 self.assertEqual(prov["artifactHash"], artifact["artifactHash"])
164
165
166 class T3IntegrationRetentionIntegrationTests(unittest.TestCase):
167 """Integration tests for retention integration (Slice 5 + T3)."""
168
169 def test_integration_t3_expired_artifact_tombstone_has_provenance(self) -> None:
170 """After TTL expiry the tombstone still exposes its provenance record."""
171
172 service = _make_service()
173 policy = {"policyClass": "ephemeral", "ttlSeconds": 60}
174 created = service.create_training_job(valid_payload("retention-prov", policy))
175 job_id = str(created["id"])
176 prov_before = service.get_provenance(job_id)
177
178 service.sweep_expired_artifacts(datetime.now(UTC) + timedelta(seconds=120))
179
180 tombstone = service.get_training_job(job_id)
181 self.assertEqual(tombstone["status"], "deleted")
182 self.assertEqual(service.list_artifacts(job_id)["artifacts"], [])
183 prov_after = service.get_provenance(job_id)
184 self.assertEqual(prov_before["jobId"], prov_after["jobId"])
185 self.assertEqual(prov_before["artifactHash"], prov_after["artifactHash"])
186 validate_provenance_record(prov_after)
187
188 def test_integration_t3_explicit_delete_wipes_provenance(self) -> None:
189 """Explicit deleteArtifact wipes provenance (Slice-5 contract preserved)."""
190
191 service = _make_service()
192 created = service.create_training_job(valid_payload("explicit-delete"))
193 job_id = str(created["id"])
194 artifact_id = str(service.list_artifacts(job_id)["artifacts"][0]["id"])
195 service.delete_artifact(job_id, artifact_id)
196 with self.assertRaises(ApiError) as raised:
197 service.get_provenance(job_id)
198 self.assertEqual(raised.exception.code, ErrorCode.NOT_FOUND)
199
200
201 if __name__ == "__main__":
202 unittest.main()
File History 1 commit
sha256:fc4c9ad652d1fff3dc508cb6ea02ee710ee6dfc4cb3761291d9900b5e029ea8a feat(slice-7): T3 dataset review lifecycle, job queue, prov… Human minor ⚠ 43 days ago