aaronrene / scooling-lab public
test_e2e.py python
130 lines 5.1 KB
Raw
sha256:91e875d4a97bb1e35f37992f803988d5713931f1782d870c371c50054574af22 Add fixture provenance retention deletion Human minor ⚠ breaking 42 days ago
1 """End-to-end tier tests for Scooling Lab HTTP training routes."""
2
3 from __future__ import annotations
4
5 import json
6 import threading
7 import unittest
8 from http.server import ThreadingHTTPServer
9 from urllib.error import HTTPError
10 from urllib.request import Request, urlopen
11
12 from scooling_lab_helpers import PROJECT_ROOT, valid_payload
13
14 from scooling_lab.api import make_handler
15 from scooling_lab.service import TrainingApiService
16 from scooling_lab.store import TrainingJobStore
17
18
19 class ScoolingLabEndToEndTests(unittest.TestCase):
20 """E2E tests across the local dependency-free HTTP API surface."""
21
22 def test_e2e_create_fetch_provenance_delete_and_verify_absence(self) -> None:
23 """A completed artifact can be fetched, deleted, and verified absent."""
24
25 service = TrainingApiService(TrainingJobStore())
26 server = ThreadingHTTPServer(("127.0.0.1", 0), make_handler(service))
27 thread = threading.Thread(target=server.serve_forever, daemon=True)
28 thread.start()
29 base_url = f"http://127.0.0.1:{server.server_port}"
30 try:
31 created = self._json_request(
32 f"{base_url}/training/jobs",
33 "POST",
34 valid_payload("e2e-provenance-delete"),
35 )
36 job_id = str(created["id"])
37 artifacts = self._json_request(
38 f"{base_url}/training/jobs/{job_id}/artifacts", "GET"
39 )
40 artifact = artifacts["artifacts"][0]
41 artifact_id = str(artifact["id"])
42 provenance = self._json_request(
43 f"{base_url}/training/jobs/{job_id}/provenance", "GET"
44 )
45 deleted_hashes = (
46 str(artifact["datasetHash"]),
47 str(artifact["artifactHash"]),
48 str(provenance["trainingConfigHash"]),
49 )
50
51 deletion = self._json_request(
52 f"{base_url}/training/jobs/{job_id}/artifacts/{artifact_id}", "DELETE"
53 )
54 job_after_delete = self._json_request(
55 f"{base_url}/training/jobs/{job_id}", "GET"
56 )
57 artifacts_after_delete = self._json_request(
58 f"{base_url}/training/jobs/{job_id}/artifacts", "GET"
59 )
60
61 self.assertTrue(deletion["verified"])
62 self.assertTrue(service.verify_deleted_artifact_absence(deleted_hashes))
63 self.assertEqual(job_after_delete["status"], "deleted")
64 self.assertEqual(artifacts_after_delete["artifacts"], [])
65 with self.assertRaises(HTTPError) as raised:
66 self._json_request(
67 f"{base_url}/training/jobs/{job_id}/provenance", "GET"
68 )
69 self.assertEqual(raised.exception.code, 404)
70 raised.exception.close()
71 finally:
72 server.shutdown()
73 server.server_close()
74 thread.join(timeout=2)
75 thread.join(timeout=2)
76
77 def _json_request(
78 self, url: str, method: str, payload: dict[str, object] | None = None
79 ) -> dict[str, object]:
80 body = None
81 headers = {"Content-Type": "application/json"}
82 if payload is not None:
83 body = json.dumps(payload).encode("utf-8")
84 request = Request(url, data=body, headers=headers, method=method)
85 with urlopen(request, timeout=5) as response:
86 decoded = json.loads(response.read().decode("utf-8"))
87 if not isinstance(decoded, dict):
88 raise AssertionError("expected JSON object")
89 return decoded
90
91 def test_e2e_http_create_poll_completed_and_list_artifacts(self) -> None:
92 """The dependency-free HTTP API completes the fake-worker fixture flow."""
93
94 service = TrainingApiService(TrainingJobStore())
95 server = ThreadingHTTPServer(("127.0.0.1", 0), make_handler(service))
96 thread = threading.Thread(target=server.serve_forever, daemon=True)
97 thread.start()
98 base_url = f"http://127.0.0.1:{server.server_port}"
99 try:
100 created = self._json_request(
101 f"{base_url}/training/jobs", "POST", valid_payload("e2e")
102 )
103 self.assertEqual(created["status"], "succeeded")
104 job_id = str(created["id"])
105
106 fetched = self._json_request(f"{base_url}/training/jobs/{job_id}", "GET")
107 self.assertEqual(fetched["status"], "succeeded")
108
109 artifacts = self._json_request(
110 f"{base_url}/training/jobs/{job_id}/artifacts", "GET"
111 )
112 self.assertEqual(len(artifacts["artifacts"]), 1)
113 self.assertEqual(artifacts["artifacts"][0]["jobId"], job_id)
114 finally:
115 server.shutdown()
116 server.server_close()
117
118 def test_e2e_ci_workflow_runs_lab_chain(self) -> None:
119 """The CI workflow contains unittest, secret scan, and BOM audit steps."""
120
121 workflow = (PROJECT_ROOT / ".github/workflows/ci.yml").read_text(
122 encoding="utf-8"
123 )
124 self.assertIn("python -m unittest discover", workflow)
125 self.assertIn("gitleaks detect", workflow)
126 self.assertIn("python -m scooling_lab.bom", workflow)
127
128
129 if __name__ == "__main__":
130 unittest.main()
File History 1 commit
sha256:91e875d4a97bb1e35f37992f803988d5713931f1782d870c371c50054574af22 Add fixture provenance retention deletion Human minor ⚠ 42 days ago