aaronrene / overseer-kit public
validate.py python
164 lines 5.8 KB
Raw
sha256:4671b7f787ddbe63ced31c895b688c77ab495653b65a730b423329f26b3c1439 feat: K1-P1 complete — agent provenance, build-verification… Sonnet 4.6 patch 54 days ago
1 """Fail-closed validator for Track N landing assets (K12)."""
2
3 from __future__ import annotations
4
5 import re
6 from dataclasses import dataclass, field
7 from pathlib import Path
8
9 import yaml
10
11 from tools.landing.schema import LandingManifest, load_manifest
12
13 # Heuristic patterns — aligned with kit security tests; not exhaustive secret scanning.
14 SECRET_PATTERNS: tuple[re.Pattern[str], ...] = (
15 re.compile(r"AKIA[0-9A-Z]{16}"),
16 re.compile(r"sk-[a-zA-Z0-9]{20,}"),
17 re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
18 re.compile(r"(?i)(?:api[_-]?key|secret|password)\s*[:=]\s*['\"][^'\"]{8,}['\"]"),
19 )
20
21 EXTERNAL_SCRIPT_RE = re.compile(
22 r"""<script[^>]+src\s*=\s*["']https?://""",
23 re.IGNORECASE,
24 )
25
26 INLINE_EVAL_RE = re.compile(r"\beval\s*\(", re.IGNORECASE)
27
28
29 @dataclass
30 class ValidationResult:
31 """Outcome of ``validate_landing``."""
32
33 ok: bool
34 errors: list[str] = field(default_factory=list)
35
36 def add(self, code: str, detail: str) -> None:
37 self.errors.append(f"{code}: {detail}")
38 self.ok = False
39
40
41 def _read(path: Path) -> str:
42 return path.read_text(encoding="utf-8")
43
44
45 def _check_html_sections(html: str, manifest: LandingManifest, result: ValidationResult) -> None:
46 for section_id in manifest.section_ids:
47 if f'id="{section_id}"' not in html and f"id='{section_id}'" not in html:
48 result.add("missing_section", section_id)
49
50
51 def _check_personas(html: str, manifest: LandingManifest, result: ValidationResult) -> None:
52 for persona_id in manifest.persona_ids:
53 marker = f'id="persona-{persona_id}"'
54 alt = f"id='persona-{persona_id}'"
55 if marker not in html and alt not in html:
56 result.add("missing_persona", persona_id)
57 continue
58 idx = html.find(marker) if marker in html else html.find(alt)
59 window = html[max(0, idx - 200) : idx + 400]
60 if not any(f"badge-{badge}" in window for badge in manifest.status_badges):
61 result.add("missing_badge", persona_id)
62
63
64 def _check_secret_leaks(text: str, rel_path: str, result: ValidationResult) -> None:
65 for pattern in SECRET_PATTERNS:
66 if pattern.search(text):
67 result.add("secret_leak", f"{rel_path} matches {pattern.pattern}")
68
69
70 def _check_html_security(html: str, rel_path: str, result: ValidationResult) -> None:
71 if EXTERNAL_SCRIPT_RE.search(html):
72 result.add("external_script", rel_path)
73 if INLINE_EVAL_RE.search(html):
74 result.add("external_script", f"{rel_path} contains eval()")
75
76
77 def _check_relative_doc_links(html: str, kit_root: Path, html_path: Path, result: ValidationResult) -> None:
78 """Resolve href='../foo.md' style links from landing pages (fragments ignored)."""
79 for match in re.finditer(r"""href=["'](\.\./[^"'#]+(?:\.md|\.yml)?)(?:#[^"']*)?["']""", html):
80 target = (html_path.parent / match.group(1)).resolve()
81 try:
82 target.relative_to(kit_root.resolve())
83 except ValueError:
84 result.add("broken_link", f"{html_path.name} -> {match.group(1)} escapes kit root")
85 continue
86 if not target.exists():
87 result.add("broken_link", f"{html_path.name} -> {match.group(1)}")
88
89
90 def validate_landing(kit_root: Path) -> ValidationResult:
91 """Validate Track N landing assets under ``kit_root`` (fail-closed)."""
92 result = ValidationResult(ok=True)
93 landing = kit_root / "docs" / "landing"
94 manifest_path = landing / "manifest.yaml"
95
96 if not manifest_path.is_file():
97 result.add("manifest_parse", "docs/landing/manifest.yaml missing")
98 return result
99
100 try:
101 manifest = load_manifest(manifest_path)
102 except (OSError, yaml.YAMLError, ValueError) as exc:
103 result.add("manifest_parse", str(exc))
104 return result
105
106 index_path = landing / "index.html"
107 scenarios_path = landing / "scenarios" / "index.html"
108 license_path = kit_root / "LICENSE"
109 security_path = kit_root / "SECURITY.md"
110
111 for path in (index_path, scenarios_path):
112 if not path.is_file():
113 result.add("missing_file", str(path.relative_to(kit_root)))
114 continue
115 html = _read(path)
116 _check_secret_leaks(html, str(path.relative_to(kit_root)), result)
117 _check_html_security(html, str(path.relative_to(kit_root)), result)
118 _check_relative_doc_links(html, kit_root, path, result)
119
120 if index_path.is_file():
121 _check_html_sections(_read(index_path), manifest, result)
122
123 if scenarios_path.is_file():
124 _check_personas(_read(scenarios_path), manifest, result)
125
126 if not license_path.is_file():
127 result.add("license", "LICENSE missing")
128 else:
129 license_text = _read(license_path)
130 if "Apache-2.0" not in license_text and "Apache License" not in license_text:
131 result.add("license", "LICENSE must reference Apache-2.0")
132 if manifest.license not in license_text and "Apache License" not in license_text:
133 result.add("license", f"manifest expects {manifest.license}")
134
135 if not security_path.is_file():
136 result.add("security", "SECURITY.md missing")
137 else:
138 security_text = _read(security_path)
139 if "Reporting a vulnerability" not in security_text:
140 result.add("security", "SECURITY.md missing disclosure section heading")
141
142 css_path = landing / "assets" / "style.css"
143 if not css_path.is_file():
144 result.add("missing_file", "docs/landing/assets/style.css")
145
146 return result
147
148
149 def main() -> int:
150 """CLI entry: ``python -m tools.landing.validate [KIT_ROOT]``."""
151 import sys
152
153 root = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else Path.cwd().resolve()
154 outcome = validate_landing(root)
155 if not outcome.ok:
156 for err in outcome.errors:
157 print(err, file=sys.stderr)
158 return 1
159 print("landing: ok")
160 return 0
161
162
163 if __name__ == "__main__":
164 raise SystemExit(main())
File History 1 commit
sha256:4671b7f787ddbe63ced31c895b688c77ab495653b65a730b423329f26b3c1439 feat: K1-P1 complete — agent provenance, build-verification… Sonnet 4.6 patch 54 days ago