gabriel / muse public
validation.py python
494 lines 21.2 KB
Raw
sha256:ac29b8ba9021514a03ab2782d92bf67671f0efa5b3b70d46f7598c5d4e923378 docs: record muse#74 Phase 4 live verification (SCR_13, SCR_14) Sonnet 5 14 days ago
1 """Trust-boundary validation primitives for Muse.
2
3 Every function in this module operates on untrusted input and either returns a
4 safe value or raises ValueError / TypeError with a descriptive message. No
5 other muse module is imported here — this module must stay at the bottom of
6 the dependency graph so it can be safely imported by every layer.
7 """
8
9 import math
10 import pathlib
11 import re
12
13 # ---------------------------------------------------------------------------
14 # Size ceilings
15 # ---------------------------------------------------------------------------
16
17 MAX_FILE_BYTES: int = 256 * 1024 * 1024 # 256 MB — per-file read cap
18 MAX_RESPONSE_BYTES: int = 64 * 1024 * 1024 # 64 MB — small-endpoint cap (refs, push, etc.)
19 MAX_SYSEX_BYTES: int = 65_536 # 64 KiB — MIDI sysex data truncation point
20 MAX_AST_BYTES: int = 2 * 1024 * 1024 # 2 MB — per-file Python AST parse cap
21
22 # Pack-format limits — applied in apply_mpack and write_object to prevent
23 # object-store poisoning and pack-bomb DoS attacks.
24 #
25 # MAX_OBJECT_WRITE_BYTES mirrors MAX_FILE_BYTES so the write and read paths
26 # enforce the same ceiling — an object that can be written can always be read.
27 #
28 # MAX_PACK_OBJECTS caps the number of blobs, snapshots, and commits that a
29 # single pack may inject in one apply_mpack call. A trillion-agent world will
30 # push many small packs, not one pack with a billion objects.
31 MAX_OBJECT_WRITE_BYTES: int = MAX_FILE_BYTES # 256 MB — per-object write cap
32 MAX_PACK_OBJECTS: int = 100_000 # maximum blobs per pack invocation
33
34 # ---------------------------------------------------------------------------
35 # Regex patterns
36 # ---------------------------------------------------------------------------
37
38 _HEX64_RE = re.compile(r"^[0-9a-f]{64}$")
39 _SHA256_PREFIXED_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
40
41 # Branch/ref names: follow Git conventions (git-check-ref-format rules).
42 # Forbidden:
43 # - All C0 control chars 0x00-0x1F (incl. null, tab, CR, LF) and DEL 0x7F
44 # - Space 0x20 (causes shell interpolation issues and log-parsing ambiguity)
45 # - Backslash (path separator on Windows; always banned for portability)
46 # - Git-banned punctuation: ~ ^ : ? * [ (ancestry, refspec, glob syntax)
47 # - Leading or trailing dot (hidden-file confusion, git pack index convention)
48 # - Consecutive dots (..) — path traversal
49 # - Leading or trailing slash — absolute-path ambiguity
50 # - Consecutive slashes (//) — normalised differently on some filesystems
51 # - Single-dot path component (/./): resolves to parent component on disk;
52 # two branch names would silently share the same ref file
53 # - Component ending in .lock: Muse uses .muse-tmp- prefix for temp files,
54 # but .lock files are a widespread VCS convention; allowing them risks
55 # ambiguity in any tooling that scans the ref directory
56 # - @{ sequence: git reflog syntax; causes parser confusion in pipelines
57 # - Bare "@": git ref notation; rejected for the same reason
58 # Allowed: forward slash (enables feature/my-branch style namespacing).
59 # Max 255 chars.
60 _BRANCH_FORBIDDEN_RE = re.compile(
61 r"[\\\x00-\x20\x7f~^:?*\[]" # backslash, all C0 + space, DEL, git punctuation
62 r"|^\." # leading dot
63 r"|\.$" # trailing dot
64 r"|\.{2,}" # consecutive dots (.., ...)
65 r"|//" # consecutive slashes
66 r"|^/" # leading slash
67 r"|/$" # trailing slash
68 r"|/\.(?:/|$)" # single-dot path component (/./ or /.<end>)
69 r"|\.lock(?:/|$)" # .lock component suffix (matches main.lock, feat/x.lock)
70 r"|@\{" # git reflog @{ sequence
71 r"|^@$" # bare @ (git HEAD shorthand)
72 )
73
74 # Valid domain plugin name: lowercase letters, digits, hyphens, underscores;
75 # must start with a lowercase letter.
76 _DOMAIN_RE = re.compile(r"^[a-z][a-z0-9_-]{0,62}$")
77
78 # Control characters to strip from terminal output.
79 # Removes all C0 (0x00-0x1F) except \t (0x09) and \n (0x0A),
80 # plus DEL (0x7F), and C1 (0x80-0x9F).
81 _CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f\x80-\x9f]")
82
83 # Glob metacharacters that must not appear in prefix arguments.
84 _GLOB_META_RE = re.compile(r"[*?\[\]{}]")
85
86 # ---------------------------------------------------------------------------
87 # ID validation
88 # ---------------------------------------------------------------------------
89
90 def validate_object_id(s: str) -> str:
91 """Return *s* unchanged if it is a canonical SHA-256 object ID.
92
93 The canonical form is ``sha256:<64 lowercase hex chars>``. Raises
94 ValueError for anything else, including bare hex — preventing
95 path-traversal attacks and catching callers that forgot the prefix.
96 """
97 if not isinstance(s, str):
98 raise TypeError(f"object_id must be str, got {type(s).__name__}")
99 if not _SHA256_PREFIXED_RE.match(s):
100 raise ValueError(
101 f"Invalid object ID {s!r}: expected 'sha256:<64 lowercase hex chars>'."
102 )
103 return s
104
105 def validate_ref_id(s: str) -> str:
106 """Return *s* unchanged if it is a canonical commit/snapshot/tag ref ID.
107
108 The canonical form is ``sha256:<64 lowercase hex chars>``. The two
109 functions exist as separate names so call-sites are self-documenting.
110 """
111 if not isinstance(s, str):
112 raise TypeError(f"ref_id must be str, got {type(s).__name__}")
113 if not _SHA256_PREFIXED_RE.match(s):
114 raise ValueError(
115 f"Invalid ref ID {s!r}: expected 'sha256:<64 lowercase hex chars>'."
116 )
117 return s
118
119 # ---------------------------------------------------------------------------
120 # Branch / repo-id validation
121 # ---------------------------------------------------------------------------
122
123 def validate_branch_name(name: str) -> str:
124 """Return *name* unchanged if it is a safe branch name.
125
126 Follows ``git check-ref-format`` conventions so Muse branch names round-trip
127 safely through any Git-backed transport:
128
129 Allowed
130 -------
131 - ASCII letters, digits, ``-``, ``_``, ``.`` (within a path component).
132 - Forward slash ``/`` as a namespace separator (``feat/my-branch``).
133
134 Rejected
135 --------
136 - All C0 control characters (0x00–0x1F), space (0x20), and DEL (0x7F).
137 Prevents terminal-injection attacks via ``for-each-ref --format text``.
138 - Backslash — path separator on Windows; banned for portability.
139 - Git-banned punctuation: ``~``, ``^``, ``:``, ``?``, ``*``, ``[``.
140 - Leading or trailing dot — hidden-file confusion.
141 - Consecutive dots (``..``) — path-traversal vector.
142 - Leading or trailing slash — absolute-path ambiguity.
143 - Consecutive slashes (``//``) — filesystem normalisation differences.
144 - Single-dot path component (``/./``): ``feat/./sub`` resolves to the same
145 inode as ``feat/sub`` on every POSIX and Windows filesystem; two branch
146 names would silently share one ref file.
147 - Any component ending in ``.lock`` (``main.lock``, ``feat/x.lock``): the
148 VCS convention reserves ``.lock`` for exclusive-lock files; allowing them
149 creates ambiguity for any tooling that scans the ref directory.
150 - The sequence ``@{`` — git reflog notation; causes parser confusion.
151 - The bare string ``@`` — git HEAD shorthand.
152 - Empty string.
153 - Names longer than 255 characters.
154 """
155 if not isinstance(name, str):
156 raise TypeError(f"branch name must be str, got {type(name).__name__}")
157 if not name:
158 raise ValueError("Branch name must not be empty.")
159 if len(name) > 255:
160 raise ValueError(
161 f"Branch name too long ({len(name)} chars); maximum is 255."
162 )
163 if _BRANCH_FORBIDDEN_RE.search(name):
164 raise ValueError(
165 f"Branch name {name!r} contains forbidden characters "
166 "(path separators, null bytes, or consecutive dots)."
167 )
168 return name
169
170 import re as _re
171 _REPO_ID_RE = _re.compile(r"^sha256:[0-9a-f]{64}$")
172
173 def validate_repo_id(repo_id: str) -> str:
174 """Return *repo_id* if it is a valid content-addressed repository ID.
175
176 Only ``sha256:<64 hex chars>`` is accepted — the canonical form produced
177 by ``muse init``.
178 """
179 if not isinstance(repo_id, str):
180 raise TypeError(f"repo_id must be str, got {type(repo_id).__name__}")
181 if not repo_id:
182 raise ValueError("repo_id must not be empty.")
183 if len(repo_id) > 255:
184 raise ValueError(f"repo_id too long ({len(repo_id)} chars).")
185 if not _REPO_ID_RE.match(repo_id):
186 raise ValueError(
187 f"repo_id must be 'sha256:<64 hex chars>', got {repo_id!r}"
188 )
189 return repo_id
190
191 def validate_domain_name(domain: str) -> str:
192 """Return *domain* if it is a valid plugin domain name."""
193 if not _DOMAIN_RE.match(domain):
194 raise ValueError(
195 f"Domain name {domain!r} is invalid. "
196 "Must start with a lowercase letter and contain only "
197 "lowercase letters, digits, hyphens, or underscores (max 63 chars)."
198 )
199 return domain
200
201 # ---------------------------------------------------------------------------
202 # Path containment
203 # ---------------------------------------------------------------------------
204
205 # Maximum length for a workspace-relative path argument (one per component;
206 # Linux PATH_MAX is 4096 — we use 4096 as the sane ceiling).
207 _MAX_WORKSPACE_PATH_LEN: int = 4096
208
209 # Detect path components that escape the root: ".." alone or leading "/".
210 # We split on both POSIX "/" and Windows "\\" for portability.
211 _DOTDOT_RE = re.compile(r"(?:^|[/\\])\.\.(?:[/\\]|$)")
212
213 def validate_workspace_path(path: str) -> str:
214 """Validate a user-supplied workspace-relative path.
215
216 Workspace paths are relative labels used for attribute lookup, ignore
217 matching, and similar pattern-only operations. They are *never* used
218 directly to open files — but we still gate them here so that:
219
220 - Null bytes, CR, and C0 control characters that could corrupt terminal
221 output or downstream shell pipelines are rejected.
222 - Absolute paths (starting with ``/`` or a drive letter) are rejected —
223 they make no sense as workspace-relative labels and almost certainly
224 indicate a confused caller or an injection attempt.
225 - Traversal sequences (``..`` as a path component) are rejected as
226 defense-in-depth, even though no file open occurs on these paths.
227 - Paths longer than 4096 characters are rejected to prevent O(N) fnmatch
228 calls from becoming a denial-of-service vector.
229
230 Returns *path* unchanged when valid. Raises :exc:`ValueError` otherwise.
231 """
232 if not isinstance(path, str):
233 raise TypeError(f"workspace path must be str, got {type(path).__name__}")
234 if not path:
235 raise ValueError("Workspace path must not be empty.")
236 if len(path) > _MAX_WORKSPACE_PATH_LEN:
237 raise ValueError(
238 f"Workspace path too long ({len(path)} chars); "
239 f"maximum is {_MAX_WORKSPACE_PATH_LEN}."
240 )
241 if "\x00" in path:
242 raise ValueError("Workspace path contains a null byte.")
243 # Reject all C0 control characters except tab (path labels should be
244 # printable; CR/LF would corrupt text output; ESC enables ANSI injection).
245 for ch in path:
246 cp = ord(ch)
247 if cp < 0x20 and cp not in (0x09,): # allow tab only
248 raise ValueError(
249 f"Workspace path contains a control character (U+{cp:04X})."
250 )
251 # Absolute paths are not workspace-relative.
252 if path.startswith("/") or path.startswith("\\") or (
253 len(path) >= 2 and path[1] == ":"
254 ):
255 raise ValueError(
256 f"Workspace path must be relative, not absolute: {path!r}"
257 )
258 # Traversal sequences — even though no filesystem open occurs, reject them
259 # as defense-in-depth and to keep output predictable for downstream tools.
260 if _DOTDOT_RE.search(path) or path == ".." or path.startswith("../") or path.startswith("..\\"):
261 raise ValueError(
262 f"Workspace path contains a traversal sequence (..): {path!r}"
263 )
264 return path
265
266 def validate_path_prefix(prefix: str) -> str:
267 """Validate a ``--path-prefix`` filter string.
268
269 Prefix filters are used as ``str.startswith()`` predicates on manifest
270 keys — they are not filesystem paths, but they must not contain:
271
272 - Null bytes (protocol confusion / injection).
273 - Control characters other than tab.
274 - Glob metacharacters (``*``, ``?``, ``[``, ``]``, ``{``, ``}``) which
275 would be meaningless here and suggest a confused caller.
276
277 Returns *prefix* unchanged when valid. Raises :exc:`ValueError` otherwise.
278 """
279 if not isinstance(prefix, str):
280 raise TypeError(f"path prefix must be str, got {type(prefix).__name__}")
281 if "\x00" in prefix:
282 raise ValueError("Path prefix contains a null byte.")
283 for ch in prefix:
284 cp = ord(ch)
285 if cp < 0x20 and cp not in (0x09,):
286 raise ValueError(
287 f"Path prefix contains a control character (U+{cp:04X})."
288 )
289 if _GLOB_META_RE.search(prefix):
290 raise ValueError(
291 f"Path prefix contains glob metacharacters: {prefix!r}"
292 )
293 return prefix
294
295 def assert_not_symlink(path: pathlib.Path, label: str = "") -> None:
296 """Raise ValueError if *path* is a symbolic link.
297
298 Used as a defence-in-depth guard on critical repository directories
299 (``.muse/``, ``.muse/objects/``, shard directories, and metadata
300 subdirectories). An attacker who replaces one of these with a symlink
301 pointing to an attacker-controlled location could redirect all subsequent
302 writes outside the repository root.
303
304 Args:
305 path: The filesystem path to check.
306 label: Human-readable name for *path* used in the error message
307 (e.g. ``".muse/"`` or ``"objects store"``).
308
309 Raises:
310 ValueError: If *path* is a symbolic link.
311 """
312 if path.is_symlink():
313 name = label or str(path)
314 raise ValueError(
315 f"Security violation: {name} is a symbolic link. "
316 "Muse refuses to use a symlinked repository directory to prevent "
317 "writes from being redirected outside the repository root."
318 )
319
320 def assert_write_inside_repo(repo_root: pathlib.Path, path: pathlib.Path) -> None:
321 """Raise ValueError if *path* resolves outside *repo_root*.
322
323 Resolves both *repo_root* and *path* (following symlinks) and verifies
324 the resolved *path* is a descendant of the resolved *repo_root*. This
325 catches TOCTOU symlink-swap attacks: even if a symlink is placed inside
326 ``.muse/`` after the startup check, the resolved destination will escape
327 the repo root and be rejected here.
328
329 Args:
330 repo_root: The repository root (parent of ``.muse/``).
331 path: The destination path to validate.
332
333 Raises:
334 ValueError: If *path* resolves to a location outside *repo_root*.
335 """
336 real_root = repo_root.resolve()
337 real_path = path.resolve()
338 if not real_path.is_relative_to(real_root):
339 raise ValueError(
340 f"Security violation: write destination {path!r} resolves to "
341 f"{real_path} which is outside the repository root {real_root}. "
342 "Possible symlink attack."
343 )
344
345 def contain_path(base: pathlib.Path, rel: str) -> pathlib.Path:
346 """Join *base* / *rel*, resolve, and assert the result stays inside *base*.
347
348 This is the central defence against zip-slip and path-traversal attacks
349 in manifest keys, rel_path arguments, and any other user-controlled path
350 component that is joined onto a trusted base directory.
351
352 Raises ValueError if the resolved path escapes *base*.
353 """
354 if not isinstance(rel, str):
355 raise TypeError(f"rel must be str, got {type(rel).__name__}")
356 if not rel:
357 raise ValueError("Relative path component must not be empty.")
358 # Absolute paths on POSIX cause pathlib to discard the base entirely.
359 joined = base / rel
360 resolved = joined.resolve()
361 base_resolved = base.resolve()
362 if not resolved.is_relative_to(base_resolved):
363 raise ValueError(
364 f"Path traversal detected: {rel!r} escapes the base directory "
365 f"{base_resolved}."
366 )
367 return resolved
368
369 # ---------------------------------------------------------------------------
370 # Glob safety
371 # ---------------------------------------------------------------------------
372
373 def sanitize_glob_prefix(prefix: str) -> str:
374 """Return *prefix* with glob metacharacters removed.
375
376 Used in _find_commit_by_prefix to prevent glob injection turning a
377 targeted lookup into an arbitrary filesystem scan.
378 """
379 return _GLOB_META_RE.sub("", prefix)
380
381 # ---------------------------------------------------------------------------
382 # Display sanitization
383 # ---------------------------------------------------------------------------
384
385 def sanitize_display(s: str) -> str:
386 """Strip terminal control characters from *s* before echoing to the user.
387
388 Preserves newline (\\n) and tab (\\t) as these are legitimate in
389 multi-line commit messages. Removes all other C0/C1 control characters
390 including ESC (0x1B), BEL (0x07), and CSI (0x9B) — the entry points for
391 ANSI/OSC terminal escape injection.
392
393 Storage is never mutated; sanitization happens only at display time.
394 """
395 return _CONTROL_CHARS_RE.sub("", s)
396
397 # Strips ALL C0 (0x00-0x1F), DEL (0x7F), and C1 (0x80-0x9F) from single-line
398 # identity strings. Unlike _CONTROL_CHARS_RE this also strips \t and \n —
399 # provenance fields are single-line; embedded newlines are never legitimate.
400 _PROVENANCE_STRIP_RE = re.compile(r"[\x00-\x1f\x7f\x80-\x9f]")
401
402 def sanitize_provenance(s: str) -> str:
403 """Sanitize a provenance string (agent_id, model_id, toolchain_id, prompt_hash).
404
405 These are single-line identity fields stored in commit records. Unlike
406 :func:`sanitize_display`, this function also strips ``\\t`` and ``\\n``
407 because provenance fields are never multi-line.
408
409 Strips all C0 control characters (0x00–0x1F), DEL (0x7F), and C1 control
410 characters (0x80–0x9F). This prevents:
411
412 - Terminal injection when provenance is rendered in ``muse log`` or
413 ``muse read`` output in future display paths.
414 - Log-line splitting if the agent_id is written to a log file.
415 - JSON structure attacks if the value is interpolated unsafely.
416 - Unicode right-to-left override (U+202E) and similar confusable chars
417 that could cause visual spoofing in terminals and web UIs.
418
419 The function does **not** truncate — callers are responsible for applying
420 the 256-character limit using ``[:_MAX_PROV]``.
421
422 Args:
423 s: Raw provenance string from environment variable or CLI flag.
424
425 Returns:
426 A copy of *s* with all C0/DEL/C1 control characters removed.
427 """
428 return _PROVENANCE_STRIP_RE.sub("", s)
429
430 _TOKEN_CTRL_RE = re.compile(r"[\x00-\x1f\x7f\x80-\x9f]")
431
432 # Maximum sensible token length — covers opaque tokens and API keys,
433 # while rejecting accidental env var pollution.
434 _MAX_TOKEN_LEN: int = 8192
435
436 def sanitize_token(raw: str) -> str | None:
437 """Validate and sanitise a token string read from an environment variable or file.
438
439 Tokens must be single-line printable ASCII/UTF-8 strings. Any value that
440 contains C0/C1 control characters (including CR and LF) would corrupt HTTP
441 headers at the transport layer. Python's ``http.client`` already blocks
442 CRLF injection at the wire, but defense-in-depth means we reject the value
443 before it ever reaches the HTTP stack so we can emit a clear diagnostic
444 rather than a confusing ``ValueError`` from deep inside ``urllib``.
445
446 Args:
447 raw: The raw token string (from env var, config file, or prompt).
448
449 Returns:
450 The token stripped of leading/trailing whitespace, or ``None`` if the
451 value is empty, too long, or contains control characters.
452 """
453 token = raw.strip()
454 if not token:
455 return None
456 if len(token) > _MAX_TOKEN_LEN:
457 return None
458 if _TOKEN_CTRL_RE.search(token):
459 return None
460 return token
461
462 # ---------------------------------------------------------------------------
463 # Numeric guards
464 # ---------------------------------------------------------------------------
465
466 def clamp_int(value: int, lo: int, hi: int, name: str = "value") -> int:
467 """Return *value* clamped to [lo, hi], raising ValueError if out of range."""
468 if not lo <= value <= hi:
469 raise ValueError(
470 f"{name} must be between {lo} and {hi}, got {value}."
471 )
472 return value
473
474 def clamp_natural(value: int, max_val: int, name: str = "value") -> int:
475 """Return *value* clamped to [0, max_val], raising ValueError if out of range.
476
477 Convenience wrapper around clamp_int for non-negative counts and limits.
478 """
479 return clamp_int(value, 0, max_val, name)
480
481 def finite_float(value: float, fallback: float, name: str = "value") -> float:
482 """Return *value* if finite, else *fallback* (and log nothing here — caller logs)."""
483 if not math.isfinite(value):
484 return fallback
485 return value
486
487 def validate_output_path(path_str: str, root: pathlib.Path) -> pathlib.Path:
488 """Resolve an --output path argument; must stay inside *root*.
489
490 Prevents a malicious agent from directing output to an arbitrary absolute
491 path such as /etc/cron.d/malicious. Raises ValueError if *path_str* resolves
492 to a location outside *root*.
493 """
494 return contain_path(root, path_str)
File History 2 commits
sha256:ac29b8ba9021514a03ab2782d92bf67671f0efa5b3b70d46f7598c5d4e923378 docs: record muse#74 Phase 4 live verification (SCR_13, SCR_14) Sonnet 5 14 days ago
sha256:2562dffa0a0822ac1bdea854f9b267843c6ce95b497a9dc5c55837c80a3ebd0a feat: domain_command_registry — Phase 1 of muse#74 (SCR_01-03) Sonnet 4.6 patch 14 days ago