orchestrator.py python
614 lines 18.4 KB
Raw
sha256:a78e7e5a8740e03315f325d19edeb3aa1b306b3337d04abbaa9a9e0f3bbeb7a1 docs: MuseHub-first before ISR #74 — staging solidify NEXT Human 12 hours ago
1 """Built-in L1 orchestrator algorithm (§K9.5)."""
2
3 from __future__ import annotations
4
5 import json
6 import os
7 import tempfile
8 from dataclasses import dataclass
9 from datetime import datetime, timezone
10 from pathlib import Path
11 from typing import Any, Callable
12
13 from adapters.config import CheckpointsConfig, OverseerConfig
14 from cli.atomic import WriteFailure, atomic_write_text
15 from cli.paths import PathEscapeError, confine_path, repo_relative
16 from tools.checkpoints.advance import compute_advance
17 from tools.checkpoints.artifact_hash import parse_artifact_sha256
18 from tools.checkpoints.argv_builder import build_verify_argv
19 from tools.checkpoints.executor import ExecResult, ScriptExecutor, SubprocessScriptExecutor
20 from tools.checkpoints.overrides import canonical_overrides_json
21 from tools.checkpoints.schema import (
22 CheckpointSchemaError,
23 load_manifest,
24 load_policy,
25 manifest_to_yaml,
26 merge_overrides,
27 render_progress,
28 resolve_template_steps,
29 )
30 from tools.checkpoints.types import (
31 ErrorToken,
32 ManifestState,
33 StepState,
34 VerifyMode,
35 VerifyStepJson,
36 VerifyStepResult,
37 )
38
39
40 @dataclass
41 class VerifyStepOptions:
42 """CLI options for verify-step."""
43
44 manifest: str | None = None
45 step_id: str | None = None
46 through_current: bool = False
47 verify_all: bool = False
48 policy: str | None = None
49 dry_run: bool = False
50 emit_json: bool = False
51
52
53 def _utc_now_iso() -> str:
54 return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
55
56
57 def _step_json_entries(
58 template_steps: list[str],
59 manifest: ManifestState,
60 selected: list[str],
61 ) -> list[dict[str, Any]]:
62 entries: list[dict[str, Any]] = []
63 for step_id in template_steps:
64 if step_id not in selected:
65 continue
66 state = manifest.steps.get(step_id, StepState())
67 entries.append(
68 {
69 "id": step_id,
70 "verified": state.verified,
71 "artifact_sha256": state.artifact_sha256,
72 }
73 )
74 return entries
75
76
77 def _result(
78 *,
79 exit_code: int,
80 mode: VerifyMode | None,
81 dry_run: bool,
82 manifest_rel: str | None,
83 steps: list[dict[str, Any]],
84 error: ErrorToken,
85 ) -> VerifyStepResult:
86 payload = VerifyStepJson(
87 ok=exit_code == 0,
88 exit_code=exit_code,
89 mode=mode,
90 dry_run=dry_run,
91 manifest=manifest_rel,
92 steps=steps,
93 error=error,
94 )
95 return VerifyStepResult(exit_code=exit_code, json_payload=payload)
96
97
98 def _resolve_mode(options: VerifyStepOptions) -> VerifyMode | None:
99 if options.step_id is not None:
100 return "step"
101 if options.through_current:
102 return "through"
103 if options.verify_all:
104 return "all"
105 return None
106
107
108 def _validate_usage(options: VerifyStepOptions) -> VerifyStepResult | None:
109 selectors = [
110 options.step_id is not None,
111 options.through_current,
112 options.verify_all,
113 ]
114 if sum(1 for x in selectors if x) != 1:
115 return _result(
116 exit_code=1,
117 mode=_resolve_mode(options),
118 dry_run=options.dry_run,
119 manifest_rel=options.manifest,
120 steps=[],
121 error="usage",
122 )
123 return None
124
125
126 def _select_through_current(
127 template_steps: list[str],
128 manifest: ManifestState,
129 ) -> tuple[list[str] | None, VerifyStepResult | None]:
130 """Return selected steps or a refusal result."""
131 unverified = [sid for sid in template_steps if not manifest.steps[sid].verified]
132 if not unverified:
133 return [], None
134 first_unverified = unverified[0]
135 current_index = template_steps.index(manifest.current_step)
136 first_index = template_steps.index(first_unverified)
137 if first_index > current_index:
138 return None, _result(
139 exit_code=11,
140 mode="through",
141 dry_run=False,
142 manifest_rel=None,
143 steps=[],
144 error="step_order",
145 )
146 end_index = current_index
147 return template_steps[first_index : end_index + 1], None
148
149
150 def _is_executable(path: Path) -> bool:
151 return path.is_file() and os.access(path, os.X_OK)
152
153
154 def run_verify_step(
155 *,
156 config: OverseerConfig,
157 repo_root: Path,
158 options: VerifyStepOptions,
159 executor: ScriptExecutor | None = None,
160 write_manifest: Callable[[Path, str], None] | None = None,
161 write_progress: Callable[[Path, str], None] | None = None,
162 ) -> VerifyStepResult:
163 """Execute the built-in verify-step orchestrator."""
164 exec_runner = executor or SubprocessScriptExecutor()
165 manifest_writer = write_manifest or (
166 lambda path, text: atomic_write_text(path, text)
167 )
168 progress_writer = write_progress or (
169 lambda path, text: atomic_write_text(path, text)
170 )
171 mode = _resolve_mode(options)
172 dry_run = options.dry_run
173
174 usage_result = _validate_usage(options)
175 if usage_result is not None:
176 usage_result.json_payload.dry_run = dry_run
177 return usage_result
178
179 checkpoints = config.checkpoints
180 if not checkpoints.enabled:
181 return _result(
182 exit_code=4,
183 mode=mode,
184 dry_run=dry_run,
185 manifest_rel=None,
186 steps=[],
187 error="refused",
188 )
189
190 if checkpoints.orchestrator:
191 return _run_consumer_override(
192 config=config,
193 repo_root=repo_root,
194 options=options,
195 mode=mode,
196 executor=exec_runner,
197 )
198
199 manifest_rel, manifest_path, manifest_code = _resolve_manifest_path(
200 repo_root, checkpoints, options.manifest
201 )
202 if manifest_code is not None:
203 return _result(
204 exit_code=manifest_code,
205 mode=mode,
206 dry_run=dry_run,
207 manifest_rel=manifest_rel,
208 steps=[],
209 error="config" if manifest_code == 2 else "refused",
210 )
211
212 policy_rel, policy_path, policy_code = _resolve_policy_path(
213 repo_root, checkpoints, options.policy
214 )
215 if policy_code is not None:
216 return _result(
217 exit_code=policy_code,
218 mode=mode,
219 dry_run=dry_run,
220 manifest_rel=manifest_rel,
221 steps=[],
222 error="config" if policy_code == 2 else "refused",
223 )
224
225 try:
226 policy = load_policy(policy_path)
227 manifest = load_manifest(manifest_path)
228 template_steps = resolve_template_steps(policy, manifest)
229 except CheckpointSchemaError:
230 return _result(
231 exit_code=2,
232 mode=mode,
233 dry_run=dry_run,
234 manifest_rel=manifest_rel,
235 steps=[],
236 error="config",
237 )
238
239 merged_overrides = merge_overrides(policy, manifest.template_id)
240
241 if mode == "step":
242 assert options.step_id is not None
243 if options.step_id not in template_steps:
244 return _result(
245 exit_code=2,
246 mode=mode,
247 dry_run=dry_run,
248 manifest_rel=manifest_rel,
249 steps=[],
250 error="config",
251 )
252 selected = [options.step_id]
253 elif mode == "through":
254 selected, refuse = _select_through_current(template_steps, manifest)
255 if refuse is not None:
256 refuse.json_payload.manifest = manifest_rel
257 refuse.json_payload.dry_run = dry_run
258 return refuse
259 assert selected is not None
260 if not selected:
261 return _result(
262 exit_code=0,
263 mode=mode,
264 dry_run=dry_run,
265 manifest_rel=manifest_rel,
266 steps=[],
267 error=None,
268 )
269 else:
270 selected = list(template_steps)
271
272 if dry_run:
273 for step_id in selected:
274 order_refuse = _check_step_order(
275 template_steps,
276 manifest,
277 step_id,
278 selected,
279 mode=mode,
280 dry_run=True,
281 manifest_rel=manifest_rel,
282 )
283 if order_refuse is not None:
284 return order_refuse
285 script_refuse = _check_script_path(
286 repo_root, policy, step_id, mode, manifest_rel, template_steps, manifest, selected
287 )
288 if script_refuse is not None:
289 script_refuse.json_payload.dry_run = True
290 return script_refuse
291 return _result(
292 exit_code=0,
293 mode=mode,
294 dry_run=True,
295 manifest_rel=manifest_rel,
296 steps=_step_json_entries(template_steps, manifest, selected),
297 error=None,
298 )
299
300 for step_id in selected:
301 order_refuse = _check_step_order(
302 template_steps,
303 manifest,
304 step_id,
305 selected,
306 mode=mode,
307 dry_run=dry_run,
308 manifest_rel=manifest_rel,
309 )
310 if order_refuse is not None:
311 return order_refuse
312
313 step_def = policy.steps[step_id]
314 try:
315 script_path = confine_path(repo_root, step_def.verify_script)
316 except PathEscapeError:
317 return _result(
318 exit_code=4,
319 mode=mode,
320 dry_run=dry_run,
321 manifest_rel=manifest_rel,
322 steps=_step_json_entries(template_steps, manifest, selected),
323 error="refused",
324 )
325 if not _is_executable(script_path):
326 return _result(
327 exit_code=4,
328 mode=mode,
329 dry_run=dry_run,
330 manifest_rel=manifest_rel,
331 steps=_step_json_entries(template_steps, manifest, selected),
332 error="refused",
333 )
334
335 script_rel = repo_relative(repo_root, script_path)
336 argv = build_verify_argv(
337 verify_script=script_rel,
338 manifest_rel=manifest_rel or "",
339 step_id=step_id,
340 policy_rel=policy_rel or "",
341 )
342
343 overrides_path: Path | None = None
344 env = {
345 "OVERSEER_REPO_ROOT": str(repo_root.resolve()),
346 "OVERSEER_CHECKPOINT_PLACEHOLDER_TOKENS": json.dumps(
347 policy.placeholder_tokens, ensure_ascii=False
348 ),
349 }
350 try:
351 with tempfile.NamedTemporaryFile(
352 mode="w",
353 encoding="utf-8",
354 delete=False,
355 suffix=".json",
356 ) as handle:
357 handle.write(canonical_overrides_json(merged_overrides))
358 overrides_path = Path(handle.name)
359 env["OVERSEER_CHECKPOINT_OVERRIDES_PATH"] = str(overrides_path.resolve())
360
361 exec_result = exec_runner.exec_argv(
362 argv,
363 cwd=str(repo_root.resolve()),
364 env=env,
365 )
366 finally:
367 if overrides_path is not None:
368 try:
369 overrides_path.unlink(missing_ok=True)
370 except OSError:
371 pass
372
373 if exec_result.exit_code != 0:
374 result = _result(
375 exit_code=10,
376 mode=mode,
377 dry_run=dry_run,
378 manifest_rel=manifest_rel,
379 steps=_step_json_entries(template_steps, manifest, selected),
380 error="verify_fail",
381 )
382 if exec_result.stderr:
383 result.stderr_extra = exec_result.stderr.decode("utf-8", errors="replace")
384 return result
385
386 sha = parse_artifact_sha256(exec_result.stdout)
387 manifest.steps[step_id].verified = True
388 manifest.steps[step_id].verified_at = _utc_now_iso()
389 manifest.steps[step_id].artifact_sha256 = sha
390
391 verified_map = {sid: manifest.steps[sid].verified for sid in template_steps}
392 manifest.current_step = compute_advance(
393 template_steps,
394 step_id,
395 verified_map,
396 manifest.current_step,
397 )
398
399 try:
400 manifest_writer(manifest_path, manifest_to_yaml(manifest))
401 if checkpoints.progress:
402 progress_path = _confine_optional(repo_root, checkpoints.progress)
403 if progress_path is not None:
404 progress_writer(
405 progress_path,
406 render_progress(manifest, template_steps),
407 )
408 except (WriteFailure, OSError):
409 return _result(
410 exit_code=5,
411 mode=mode,
412 dry_run=dry_run,
413 manifest_rel=manifest_rel,
414 steps=_step_json_entries(template_steps, manifest, selected),
415 error="io",
416 )
417
418 return _result(
419 exit_code=0,
420 mode=mode,
421 dry_run=dry_run,
422 manifest_rel=manifest_rel,
423 steps=_step_json_entries(template_steps, manifest, selected),
424 error=None,
425 )
426
427
428 def _confine_optional(repo_root: Path, user_path: str) -> Path | None:
429 try:
430 return confine_path(repo_root, user_path)
431 except PathEscapeError:
432 return None
433
434
435 def _resolve_manifest_path(
436 repo_root: Path,
437 checkpoints: CheckpointsConfig,
438 manifest_arg: str | None,
439 ) -> tuple[str | None, Path | None, int | None]:
440 rel = manifest_arg or checkpoints.active_manifest
441 if rel is None or not str(rel).strip():
442 return None, None, 2
443 try:
444 path = confine_path(repo_root, rel)
445 except PathEscapeError:
446 return rel, None, 4
447 if not path.is_file():
448 return rel, None, 4
449 return repo_relative(repo_root, path), path, None
450
451
452 def _resolve_policy_path(
453 repo_root: Path,
454 checkpoints: CheckpointsConfig,
455 policy_arg: str | None,
456 ) -> tuple[str | None, Path | None, int | None]:
457 rel = policy_arg or checkpoints.policy
458 if rel is None or not str(rel).strip():
459 return None, None, 2
460 try:
461 path = confine_path(repo_root, rel)
462 except PathEscapeError:
463 return rel, None, 4
464 if not path.is_file():
465 return rel, None, 4
466 return repo_relative(repo_root, path), path, None
467
468
469 def _check_step_order(
470 template_steps: list[str],
471 manifest: ManifestState,
472 step_id: str,
473 selected: list[str],
474 *,
475 mode: VerifyMode | None,
476 dry_run: bool,
477 manifest_rel: str | None,
478 ) -> VerifyStepResult | None:
479 """Refuse when a prior template step is not verified (§K9.5 step 6a)."""
480 step_index = template_steps.index(step_id)
481 selected_index = selected.index(step_id)
482 earlier_in_run = set(selected[:selected_index])
483 for prior_id in template_steps[:step_index]:
484 if manifest.steps[prior_id].verified:
485 continue
486 if prior_id in earlier_in_run:
487 continue
488 return _result(
489 exit_code=11,
490 mode=mode,
491 dry_run=dry_run,
492 manifest_rel=manifest_rel,
493 steps=_step_json_entries(template_steps, manifest, selected),
494 error="step_order",
495 )
496 return None
497
498
499 def _check_script_path(
500 repo_root: Path,
501 policy,
502 step_id: str,
503 mode: VerifyMode | None,
504 manifest_rel: str | None,
505 template_steps: list[str],
506 manifest: ManifestState,
507 selected: list[str],
508 ) -> VerifyStepResult | None:
509 step_def = policy.steps[step_id]
510 try:
511 script_path = confine_path(repo_root, step_def.verify_script)
512 except PathEscapeError:
513 return _result(
514 exit_code=4,
515 mode=mode,
516 dry_run=True,
517 manifest_rel=manifest_rel,
518 steps=_step_json_entries(template_steps, manifest, selected),
519 error="refused",
520 )
521 if not _is_executable(script_path):
522 return _result(
523 exit_code=4,
524 mode=mode,
525 dry_run=True,
526 manifest_rel=manifest_rel,
527 steps=_step_json_entries(template_steps, manifest, selected),
528 error="refused",
529 )
530 return None
531
532
533 def _run_consumer_override(
534 *,
535 config: OverseerConfig,
536 repo_root: Path,
537 options: VerifyStepOptions,
538 mode: VerifyMode | None,
539 executor: ScriptExecutor,
540 ) -> VerifyStepResult:
541 """Invoke consumer orchestrator override (§K9.5)."""
542 orchestrator_rel = config.checkpoints.orchestrator
543 assert orchestrator_rel
544 try:
545 script_path = confine_path(repo_root, orchestrator_rel)
546 except PathEscapeError:
547 return _result(
548 exit_code=4,
549 mode=mode,
550 dry_run=options.dry_run,
551 manifest_rel=options.manifest,
552 steps=[],
553 error="refused",
554 )
555 if not _is_executable(script_path):
556 return _result(
557 exit_code=4,
558 mode=mode,
559 dry_run=options.dry_run,
560 manifest_rel=options.manifest,
561 steps=[],
562 error="refused",
563 )
564 argv = [repo_relative(repo_root, script_path)]
565 # Pass through verify-step flags after script name.
566 if options.manifest:
567 argv.extend(["--manifest", options.manifest])
568 if options.step_id:
569 argv.extend(["--step", options.step_id])
570 if options.through_current:
571 argv.append("--through")
572 argv.append("current")
573 if options.verify_all:
574 argv.append("--all")
575 if options.policy:
576 argv.extend(["--policy", options.policy])
577 if options.dry_run:
578 argv.append("--dry-run")
579 if options.emit_json:
580 argv.append("--json")
581
582 env = {"OVERSEER_REPO_ROOT": str(repo_root.resolve())}
583 exec_result = executor.exec_argv(
584 argv,
585 cwd=str(repo_root.resolve()),
586 env=env,
587 )
588 if options.emit_json:
589 # Override must emit JSON; pass through stdout unchanged via stderr_extra hack — CLI handles.
590 return VerifyStepResult(
591 exit_code=exec_result.exit_code,
592 json_payload=VerifyStepJson(
593 ok=exec_result.exit_code == 0,
594 exit_code=exec_result.exit_code,
595 mode=mode,
596 dry_run=options.dry_run,
597 manifest=options.manifest,
598 steps=[],
599 error=None if exec_result.exit_code == 0 else "refused",
600 ),
601 stderr_extra=exec_result.stderr.decode("utf-8", errors="replace"),
602 )
603 return VerifyStepResult(
604 exit_code=exec_result.exit_code,
605 json_payload=VerifyStepJson(
606 ok=exec_result.exit_code == 0,
607 exit_code=exec_result.exit_code,
608 mode=mode,
609 dry_run=options.dry_run,
610 manifest=options.manifest,
611 steps=[],
612 error=None,
613 ),
614 )
File History 2 commits
sha256:a78e7e5a8740e03315f325d19edeb3aa1b306b3337d04abbaa9a9e0f3bbeb7a1 docs: MuseHub-first before ISR #74 — staging solidify NEXT Human 12 hours ago
sha256:4671b7f787ddbe63ced31c895b688c77ab495653b65a730b423329f26b3c1439 feat: K1-P1 complete — agent provenance, build-verification… Sonnet 4.6 patch 52 days ago