validation.py python
518 lines 21.2 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 33 days ago
1 """Trust-boundary validation primitives for Muse.
2
3 Every function in this module operates on untrusted input and either returns a
4 safe value or raises ValueError / TypeError with a descriptive message. No
5 other muse module is imported here — this module must stay at the bottom of
6 the dependency graph so it can be safely imported by every layer.
7 """
8
9 from __future__ import annotations
10
11 import math
12 import pathlib
13 import re
14
15 # ---------------------------------------------------------------------------
16 # Size ceilings
17 # ---------------------------------------------------------------------------
18
19 MAX_FILE_BYTES: int = 256 * 1024 * 1024 # 256 MB — per-file read cap
20 MAX_RESPONSE_BYTES: int = 64 * 1024 * 1024 # 64 MB — small-endpoint cap (refs, push, etc.)
21 MAX_FETCH_BYTES: int = 2 * 1024 * 1024 * 1024 # 2 GiB — fetch/clone streaming cap
22 MAX_SYSEX_BYTES: int = 65_536 # 64 KiB — MIDI sysex data truncation point
23 MAX_AST_BYTES: int = 2 * 1024 * 1024 # 2 MB — per-file Python AST parse cap
24
25 # Pack-format limits — applied in apply_pack and write_object to prevent
26 # object-store poisoning and pack-bomb DoS attacks.
27 #
28 # MAX_OBJECT_WRITE_BYTES mirrors MAX_FILE_BYTES so the write and read paths
29 # enforce the same ceiling — an object that can be written can always be read.
30 #
31 # MAX_PACK_OBJECTS caps the number of blobs, snapshots, and commits that a
32 # single pack may inject in one apply_pack call. A trillion-agent world will
33 # push many small packs, not one pack with a billion objects.
34 MAX_OBJECT_WRITE_BYTES: int = MAX_FILE_BYTES # 256 MB — per-object write cap
35 MAX_PACK_OBJECTS: int = 100_000 # maximum blobs per pack invocation
36
37 # ---------------------------------------------------------------------------
38 # Regex patterns
39 # ---------------------------------------------------------------------------
40
41 _HEX64_RE = re.compile(r"^[0-9a-f]{64}$")
42
43 # Branch/ref names: follow Git conventions (git-check-ref-format rules).
44 # Forbidden:
45 # - All C0 control chars 0x00-0x1F (incl. null, tab, CR, LF) and DEL 0x7F
46 # - Space 0x20 (causes shell interpolation issues and log-parsing ambiguity)
47 # - Backslash (path separator on Windows; always banned for portability)
48 # - Git-banned punctuation: ~ ^ : ? * [ (ancestry, refspec, glob syntax)
49 # - Leading or trailing dot (hidden-file confusion, git pack index convention)
50 # - Consecutive dots (..) — path traversal
51 # - Leading or trailing slash — absolute-path ambiguity
52 # - Consecutive slashes (//) — normalised differently on some filesystems
53 # - Single-dot path component (/./): resolves to parent component on disk;
54 # two branch names would silently share the same ref file
55 # - Component ending in .lock: Muse uses .muse-tmp- prefix for temp files,
56 # but .lock files are a widespread VCS convention; allowing them risks
57 # ambiguity in any tooling that scans the ref directory
58 # - @{ sequence: git reflog syntax; causes parser confusion in pipelines
59 # - Bare "@": git ref notation; rejected for the same reason
60 # Allowed: forward slash (enables feature/my-branch style namespacing).
61 # Max 255 chars.
62 _BRANCH_FORBIDDEN_RE = re.compile(
63 r"[\\\x00-\x20\x7f~^:?*\[]" # backslash, all C0 + space, DEL, git punctuation
64 r"|^\." # leading dot
65 r"|\.$" # trailing dot
66 r"|\.{2,}" # consecutive dots (.., ...)
67 r"|//" # consecutive slashes
68 r"|^/" # leading slash
69 r"|/$" # trailing slash
70 r"|/\.(?:/|$)" # single-dot path component (/./ or /.<end>)
71 r"|\.lock(?:/|$)" # .lock component suffix (matches main.lock, feat/x.lock)
72 r"|@\{" # git reflog @{ sequence
73 r"|^@$" # bare @ (git HEAD shorthand)
74 )
75
76 # Valid domain plugin name: lowercase letters, digits, hyphens, underscores;
77 # must start with a lowercase letter.
78 _DOMAIN_RE = re.compile(r"^[a-z][a-z0-9_-]{0,62}$")
79
80 # Control characters to strip from terminal output.
81 # Removes all C0 (0x00-0x1F) except \t (0x09) and \n (0x0A),
82 # plus DEL (0x7F), and C1 (0x80-0x9F).
83 _CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f\x80-\x9f]")
84
85 # Glob metacharacters that must not appear in prefix arguments.
86 _GLOB_META_RE = re.compile(r"[*?\[\]{}]")
87
88 # ---------------------------------------------------------------------------
89 # ID validation
90 # ---------------------------------------------------------------------------
91
92
93 def validate_object_id(s: str) -> str:
94 """Return *s* unchanged if it is a valid SHA-256 hex string (64 lowercase hex chars).
95
96 Raises ValueError for anything else, preventing path-traversal attacks
97 built from crafted object IDs.
98 """
99 if not isinstance(s, str):
100 raise TypeError(f"object_id must be str, got {type(s).__name__}")
101 if not _HEX64_RE.match(s):
102 raise ValueError(
103 f"Invalid object ID {s!r}: expected exactly 64 lowercase hex characters."
104 )
105 return s
106
107
108 def validate_ref_id(s: str) -> str:
109 """Return *s* unchanged if it is a valid commit/snapshot/tag ID.
110
111 Uses the same 64-char hex rule as object IDs; the two functions exist as
112 separate names so call-sites are self-documenting.
113 """
114 if not isinstance(s, str):
115 raise TypeError(f"ref_id must be str, got {type(s).__name__}")
116 if not _HEX64_RE.match(s):
117 raise ValueError(
118 f"Invalid ref ID {s!r}: expected exactly 64 lowercase hex characters."
119 )
120 return s
121
122
123 # ---------------------------------------------------------------------------
124 # Branch / repo-id validation
125 # ---------------------------------------------------------------------------
126
127
128 def validate_branch_name(name: str) -> str:
129 """Return *name* unchanged if it is a safe branch name.
130
131 Follows ``git check-ref-format`` conventions so Muse branch names round-trip
132 safely through any Git-backed transport:
133
134 Allowed
135 -------
136 - ASCII letters, digits, ``-``, ``_``, ``.`` (within a path component).
137 - Forward slash ``/`` as a namespace separator (``feat/my-branch``).
138
139 Rejected
140 --------
141 - All C0 control characters (0x00–0x1F), space (0x20), and DEL (0x7F).
142 Prevents terminal-injection attacks via ``for-each-ref --format text``.
143 - Backslash — path separator on Windows; banned for portability.
144 - Git-banned punctuation: ``~``, ``^``, ``:``, ``?``, ``*``, ``[``.
145 - Leading or trailing dot — hidden-file confusion.
146 - Consecutive dots (``..``) — path-traversal vector.
147 - Leading or trailing slash — absolute-path ambiguity.
148 - Consecutive slashes (``//``) — filesystem normalisation differences.
149 - Single-dot path component (``/./``): ``feat/./sub`` resolves to the same
150 inode as ``feat/sub`` on every POSIX and Windows filesystem; two branch
151 names would silently share one ref file.
152 - Any component ending in ``.lock`` (``main.lock``, ``feat/x.lock``): the
153 VCS convention reserves ``.lock`` for exclusive-lock files; allowing them
154 creates ambiguity for any tooling that scans the ref directory.
155 - The sequence ``@{`` — git reflog notation; causes parser confusion.
156 - The bare string ``@`` — git HEAD shorthand.
157 - Empty string.
158 - Names longer than 255 characters.
159 """
160 if not isinstance(name, str):
161 raise TypeError(f"branch name must be str, got {type(name).__name__}")
162 if not name:
163 raise ValueError("Branch name must not be empty.")
164 if len(name) > 255:
165 raise ValueError(
166 f"Branch name too long ({len(name)} chars); maximum is 255."
167 )
168 if _BRANCH_FORBIDDEN_RE.search(name):
169 raise ValueError(
170 f"Branch name {name!r} contains forbidden characters "
171 "(path separators, null bytes, or consecutive dots)."
172 )
173 return name
174
175
176 def validate_repo_id(repo_id: str) -> str:
177 """Return *repo_id* if it contains no path-traversal components.
178
179 repo_id values are UUIDs in normal operation. We enforce that they
180 contain no path separators or dot-sequences rather than enforcing UUID
181 format strictly, to allow future flexibility.
182 """
183 if not isinstance(repo_id, str):
184 raise TypeError(f"repo_id must be str, got {type(repo_id).__name__}")
185 if not repo_id:
186 raise ValueError("repo_id must not be empty.")
187 if len(repo_id) > 255:
188 raise ValueError(f"repo_id too long ({len(repo_id)} chars).")
189 if _BRANCH_FORBIDDEN_RE.search(repo_id):
190 raise ValueError(
191 f"repo_id {repo_id!r} contains forbidden characters."
192 )
193 return repo_id
194
195
196 def validate_domain_name(domain: str) -> str:
197 """Return *domain* if it is a valid plugin domain name."""
198 if not _DOMAIN_RE.match(domain):
199 raise ValueError(
200 f"Domain name {domain!r} is invalid. "
201 "Must start with a lowercase letter and contain only "
202 "lowercase letters, digits, hyphens, or underscores (max 63 chars)."
203 )
204 return domain
205
206
207 # ---------------------------------------------------------------------------
208 # Path containment
209 # ---------------------------------------------------------------------------
210
211 # Maximum length for a workspace-relative path argument (one per component;
212 # Linux PATH_MAX is 4096 — we use 4096 as the sane ceiling).
213 _MAX_WORKSPACE_PATH_LEN: int = 4096
214
215 # Detect path components that escape the root: ".." alone or leading "/".
216 # We split on both POSIX "/" and Windows "\\" for portability.
217 _DOTDOT_RE = re.compile(r"(?:^|[/\\])\.\.(?:[/\\]|$)")
218
219
220 def validate_workspace_path(path: str) -> str:
221 """Validate a user-supplied workspace-relative path.
222
223 Workspace paths are relative labels used for attribute lookup, ignore
224 matching, and similar pattern-only operations. They are *never* used
225 directly to open files — but we still gate them here so that:
226
227 - Null bytes, CR, and C0 control characters that could corrupt terminal
228 output or downstream shell pipelines are rejected.
229 - Absolute paths (starting with ``/`` or a drive letter) are rejected —
230 they make no sense as workspace-relative labels and almost certainly
231 indicate a confused caller or an injection attempt.
232 - Traversal sequences (``..`` as a path component) are rejected as
233 defense-in-depth, even though no file open occurs on these paths.
234 - Paths longer than 4096 characters are rejected to prevent O(N) fnmatch
235 calls from becoming a denial-of-service vector.
236
237 Returns *path* unchanged when valid. Raises :exc:`ValueError` otherwise.
238 """
239 if not isinstance(path, str):
240 raise TypeError(f"workspace path must be str, got {type(path).__name__}")
241 if not path:
242 raise ValueError("Workspace path must not be empty.")
243 if len(path) > _MAX_WORKSPACE_PATH_LEN:
244 raise ValueError(
245 f"Workspace path too long ({len(path)} chars); "
246 f"maximum is {_MAX_WORKSPACE_PATH_LEN}."
247 )
248 if "\x00" in path:
249 raise ValueError("Workspace path contains a null byte.")
250 # Reject all C0 control characters except tab (path labels should be
251 # printable; CR/LF would corrupt text output; ESC enables ANSI injection).
252 for ch in path:
253 cp = ord(ch)
254 if cp < 0x20 and cp not in (0x09,): # allow tab only
255 raise ValueError(
256 f"Workspace path contains a control character (U+{cp:04X})."
257 )
258 # Absolute paths are not workspace-relative.
259 if path.startswith("/") or path.startswith("\\") or (
260 len(path) >= 2 and path[1] == ":"
261 ):
262 raise ValueError(
263 f"Workspace path must be relative, not absolute: {path!r}"
264 )
265 # Traversal sequences — even though no filesystem open occurs, reject them
266 # as defense-in-depth and to keep output predictable for downstream tools.
267 if _DOTDOT_RE.search(path) or path == ".." or path.startswith("../") or path.startswith("..\\"):
268 raise ValueError(
269 f"Workspace path contains a traversal sequence (..): {path!r}"
270 )
271 return path
272
273
274 def validate_path_prefix(prefix: str) -> str:
275 """Validate a ``--path-prefix`` filter string.
276
277 Prefix filters are used as ``str.startswith()`` predicates on manifest
278 keys — they are not filesystem paths, but they must not contain:
279
280 - Null bytes (protocol confusion / injection).
281 - Control characters other than tab.
282 - Glob metacharacters (``*``, ``?``, ``[``, ``]``, ``{``, ``}``) which
283 would be meaningless here and suggest a confused caller.
284
285 Returns *prefix* unchanged when valid. Raises :exc:`ValueError` otherwise.
286 """
287 if not isinstance(prefix, str):
288 raise TypeError(f"path prefix must be str, got {type(prefix).__name__}")
289 if "\x00" in prefix:
290 raise ValueError("Path prefix contains a null byte.")
291 for ch in prefix:
292 cp = ord(ch)
293 if cp < 0x20 and cp not in (0x09,):
294 raise ValueError(
295 f"Path prefix contains a control character (U+{cp:04X})."
296 )
297 if _GLOB_META_RE.search(prefix):
298 raise ValueError(
299 f"Path prefix contains glob metacharacters: {prefix!r}"
300 )
301 return prefix
302
303
304 def assert_not_symlink(path: pathlib.Path, label: str = "") -> None:
305 """Raise ValueError if *path* is a symbolic link.
306
307 Used as a defence-in-depth guard on critical repository directories
308 (``.muse/``, ``.muse/objects/``, shard directories, and metadata
309 subdirectories). An attacker who replaces one of these with a symlink
310 pointing to an attacker-controlled location could redirect all subsequent
311 writes outside the repository root.
312
313 Args:
314 path: The filesystem path to check.
315 label: Human-readable name for *path* used in the error message
316 (e.g. ``".muse/"`` or ``"objects store"``).
317
318 Raises:
319 ValueError: If *path* is a symbolic link.
320 """
321 if path.is_symlink():
322 name = label or str(path)
323 raise ValueError(
324 f"Security violation: {name} is a symbolic link. "
325 "Muse refuses to use a symlinked repository directory to prevent "
326 "writes from being redirected outside the repository root."
327 )
328
329
330 def assert_write_inside_repo(repo_root: pathlib.Path, path: pathlib.Path) -> None:
331 """Raise ValueError if *path* resolves outside *repo_root*.
332
333 Resolves both *repo_root* and *path* (following symlinks) and verifies
334 the resolved *path* is a descendant of the resolved *repo_root*. This
335 catches TOCTOU symlink-swap attacks: even if a symlink is placed inside
336 ``.muse/`` after the startup check, the resolved destination will escape
337 the repo root and be rejected here.
338
339 Args:
340 repo_root: The repository root (parent of ``.muse/``).
341 path: The destination path to validate.
342
343 Raises:
344 ValueError: If *path* resolves to a location outside *repo_root*.
345 """
346 real_root = repo_root.resolve()
347 real_path = path.resolve()
348 if not real_path.is_relative_to(real_root):
349 raise ValueError(
350 f"Security violation: write destination {path!r} resolves to "
351 f"{real_path} which is outside the repository root {real_root}. "
352 "Possible symlink attack."
353 )
354
355
356 def contain_path(base: pathlib.Path, rel: str) -> pathlib.Path:
357 """Join *base* / *rel*, resolve, and assert the result stays inside *base*.
358
359 This is the central defence against zip-slip and path-traversal attacks
360 in manifest keys, rel_path arguments, and any other user-controlled path
361 component that is joined onto a trusted base directory.
362
363 Raises ValueError if the resolved path escapes *base*.
364 """
365 if not isinstance(rel, str):
366 raise TypeError(f"rel must be str, got {type(rel).__name__}")
367 if not rel:
368 raise ValueError("Relative path component must not be empty.")
369 # Absolute paths on POSIX cause pathlib to discard the base entirely.
370 joined = base / rel
371 resolved = joined.resolve()
372 base_resolved = base.resolve()
373 if not resolved.is_relative_to(base_resolved):
374 raise ValueError(
375 f"Path traversal detected: {rel!r} escapes the base directory "
376 f"{base_resolved}."
377 )
378 return resolved
379
380
381 # ---------------------------------------------------------------------------
382 # Glob safety
383 # ---------------------------------------------------------------------------
384
385
386 def sanitize_glob_prefix(prefix: str) -> str:
387 """Return *prefix* with glob metacharacters removed.
388
389 Used in _find_commit_by_prefix to prevent glob injection turning a
390 targeted lookup into an arbitrary filesystem scan.
391 """
392 return _GLOB_META_RE.sub("", prefix)
393
394
395 # ---------------------------------------------------------------------------
396 # Display sanitization
397 # ---------------------------------------------------------------------------
398
399
400 def sanitize_display(s: str) -> str:
401 """Strip terminal control characters from *s* before echoing to the user.
402
403 Preserves newline (\\n) and tab (\\t) as these are legitimate in
404 multi-line commit messages. Removes all other C0/C1 control characters
405 including ESC (0x1B), BEL (0x07), and CSI (0x9B) — the entry points for
406 ANSI/OSC terminal escape injection.
407
408 Storage is never mutated; sanitization happens only at display time.
409 """
410 return _CONTROL_CHARS_RE.sub("", s)
411
412
413 # Strips ALL C0 (0x00-0x1F), DEL (0x7F), and C1 (0x80-0x9F) from single-line
414 # identity strings. Unlike _CONTROL_CHARS_RE this also strips \t and \n —
415 # provenance fields are single-line; embedded newlines are never legitimate.
416 _PROVENANCE_STRIP_RE = re.compile(r"[\x00-\x1f\x7f\x80-\x9f]")
417
418
419 def sanitize_provenance(s: str) -> str:
420 """Sanitize a provenance string (agent_id, model_id, toolchain_id, prompt_hash).
421
422 These are single-line identity fields stored in commit records. Unlike
423 :func:`sanitize_display`, this function also strips ``\\t`` and ``\\n``
424 because provenance fields are never multi-line.
425
426 Strips all C0 control characters (0x00–0x1F), DEL (0x7F), and C1 control
427 characters (0x80–0x9F). This prevents:
428
429 - Terminal injection when provenance is rendered in ``muse log`` or
430 ``muse show`` output in future display paths.
431 - Log-line splitting if the agent_id is written to a log file.
432 - JSON structure attacks if the value is interpolated unsafely.
433 - Unicode right-to-left override (U+202E) and similar confusable chars
434 that could cause visual spoofing in terminals and web UIs.
435
436 The function does **not** truncate — callers are responsible for applying
437 the 256-character limit using ``[:_MAX_PROV]``.
438
439 Args:
440 s: Raw provenance string from environment variable or CLI flag.
441
442 Returns:
443 A copy of *s* with all C0/DEL/C1 control characters removed.
444 """
445 return _PROVENANCE_STRIP_RE.sub("", s)
446
447
448 _TOKEN_CTRL_RE = re.compile(r"[\x00-\x1f\x7f\x80-\x9f]")
449
450 # Maximum sensible token length — covers opaque tokens and API keys,
451 # while rejecting accidental env var pollution.
452 _MAX_TOKEN_LEN: int = 8192
453
454
455 def sanitize_token(raw: str) -> str | None:
456 """Validate and sanitise a token string read from an environment variable or file.
457
458 Tokens must be single-line printable ASCII/UTF-8 strings. Any value that
459 contains C0/C1 control characters (including CR and LF) would corrupt HTTP
460 headers at the transport layer. Python's ``http.client`` already blocks
461 CRLF injection at the wire, but defense-in-depth means we reject the value
462 before it ever reaches the HTTP stack so we can emit a clear diagnostic
463 rather than a confusing ``ValueError`` from deep inside ``urllib``.
464
465 Args:
466 raw: The raw token string (from env var, config file, or prompt).
467
468 Returns:
469 The token stripped of leading/trailing whitespace, or ``None`` if the
470 value is empty, too long, or contains control characters.
471 """
472 token = raw.strip()
473 if not token:
474 return None
475 if len(token) > _MAX_TOKEN_LEN:
476 return None
477 if _TOKEN_CTRL_RE.search(token):
478 return None
479 return token
480
481
482 # ---------------------------------------------------------------------------
483 # Numeric guards
484 # ---------------------------------------------------------------------------
485
486
487 def clamp_int(value: int, lo: int, hi: int, name: str = "value") -> int:
488 """Return *value* clamped to [lo, hi], raising ValueError if out of range."""
489 if not lo <= value <= hi:
490 raise ValueError(
491 f"{name} must be between {lo} and {hi}, got {value}."
492 )
493 return value
494
495
496 def clamp_natural(value: int, max_val: int, name: str = "value") -> int:
497 """Return *value* clamped to [0, max_val], raising ValueError if out of range.
498
499 Convenience wrapper around clamp_int for non-negative counts and limits.
500 """
501 return clamp_int(value, 0, max_val, name)
502
503
504 def finite_float(value: float, fallback: float, name: str = "value") -> float:
505 """Return *value* if finite, else *fallback* (and log nothing here — caller logs)."""
506 if not math.isfinite(value):
507 return fallback
508 return value
509
510
511 def validate_output_path(path_str: str, root: pathlib.Path) -> pathlib.Path:
512 """Resolve an --output path argument; must stay inside *root*.
513
514 Prevents a malicious agent from directing output to an arbitrary absolute
515 path such as /etc/cron.d/evil. Raises ValueError if *path_str* resolves
516 to a location outside *root*.
517 """
518 return contain_path(root, path_str)
File History 2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 33 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 33 days ago