gabriel / muse public
events.py python
452 lines 19.7 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
1 """Knowtation memory event schema — Phase 4.1, locked at v1.0.0.
2
3 This module is the **canonical Python source of truth** for the 15 memory event
4 kinds that Knowtation emits and that Muse persists in commit metadata. It
5 mirrors the JavaScript enum in :file:`/Users/aaronrenecarvajal/knowtation/lib/memory-event.mjs`
6 field-for-field and value-for-value. Any drift between the two is a
7 correctness bug and is detected by the data-integrity tests in
8 :file:`tests/test_knowtation_events.py` (Tier 5).
9
10 Schema lock policy
11 ------------------
12 :data:`EVENTS_SCHEMA_VERSION` is **immutable once shipped**. Bumping it
13 requires a written migration path that converts existing commit metadata
14 records. Renaming, removing, or reordering :class:`MemoryEventKind` members
15 is treated as a breaking schema change and **must** be paired with a version
16 bump and a migration. Adding a new kind value is also a breaking change
17 because :data:`EVENTS_SCHEMA_HASH` is content-addressed over the sorted value
18 list — readers pinned to ``EVENTS_SCHEMA_HASH`` would refuse the new value.
19
20 Sensitive-data handling
21 -----------------------
22 :func:`MemoryEventRecord.__post_init__` rejects any ``data`` payload that
23 contains a key matching the same regular expression used by the JavaScript
24 ``hasSensitiveKeys`` function: ``api[_-]?key``, ``secret``, ``password``,
25 ``token``, ``credential``, ``authorization``, ``bearer``, ``private[_-]?key``,
26 case-insensitive. The scan is recursive into nested dicts and lists up to a
27 maximum depth of 8, matching the JS implementation byte-for-byte. Beyond that
28 depth the scan returns ``False`` rather than recursing further, which is a
29 deliberate trade-off between thoroughness and amplification-attack resistance.
30
31 Public API
32 ----------
33 * :data:`EVENTS_SCHEMA_VERSION` — immutable schema version string.
34 * :data:`EVENTS_SCHEMA_HASH` — SHA-256 of the canonical sorted value list.
35 * :class:`MemoryEventKind` — :class:`enum.StrEnum` of the 15 canonical kinds.
36 * :class:`MemoryEventRecord` — frozen dataclass holding one validated event.
37 * :func:`is_valid_event_kind` — non-raising predicate for runtime validation.
38 * :func:`from_dict` — strict deserialiser; raises :class:`ValueError`.
39 * :func:`to_dict` — minimal serialiser; omits ``None``-valued fields.
40
41 Round-trip identity
42 -------------------
43 For every well-formed :class:`MemoryEventRecord` ``r``,
44 ``from_dict(to_dict(r)) == r`` holds. The serialiser emits Python field names
45 (``event_id``, ``kind``, ...) rather than the JavaScript-side names
46 (``id``, ``type``, ...) so the canonical Python dict shape is unambiguous.
47 Cross-runtime adapters live outside this module so the schema lock is
48 single-purpose.
49 """
50
51 from __future__ import annotations
52
53 import copy
54 import hashlib
55 import json
56 import re
57 from dataclasses import dataclass, field
58 from datetime import datetime
59 from enum import StrEnum
60 from typing import Any, Final, Literal
61
62 # ──────────────────────────────────────────────────────────────────────────────
63 # Schema version — IMMUTABLE
64 # ──────────────────────────────────────────────────────────────────────────────
65
66 #: The locked schema version for the memory-event contract. Bumping this
67 #: constant requires a paired migration script for prior commit metadata.
68 EVENTS_SCHEMA_VERSION: Final[str] = "1.0.0"
69
70
71 # ──────────────────────────────────────────────────────────────────────────────
72 # Canonical event-kind enumeration
73 # ──────────────────────────────────────────────────────────────────────────────
74
75
76 class MemoryEventKind(StrEnum):
77 """The 15 canonical memory event kinds for the Knowtation domain.
78
79 Mirrored from ``MEMORY_EVENT_TYPES`` in
80 :file:`knowtation/lib/memory-event.mjs`. The string *value* of each
81 member is the wire-format name that appears in Muse commit metadata;
82 the Python member name is uppercase to satisfy PEP 8 and (for ``IMPORT``)
83 to avoid colliding with the Python keyword.
84
85 Use the value (e.g. ``MemoryEventKind.SEARCH``) directly in string
86 contexts — :class:`enum.StrEnum` makes ``str(MemoryEventKind.SEARCH) ==
87 "search"`` and ``MemoryEventKind.SEARCH == "search"`` both true.
88 """
89
90 SEARCH = "search"
91 EXPORT = "export"
92 WRITE = "write"
93 IMPORT = "import"
94 INDEX = "index"
95 PROPOSE = "propose"
96 AGENT_INTERACTION = "agent_interaction"
97 CAPTURE = "capture"
98 ERROR = "error"
99 SESSION_SUMMARY = "session_summary"
100 USER = "user"
101 CONSOLIDATION = "consolidation"
102 CONSOLIDATION_PASS = "consolidation_pass"
103 MAINTENANCE = "maintenance"
104 INSIGHT = "insight"
105
106
107 #: Frozen set of every valid wire-format kind value. Used by
108 #: :func:`is_valid_event_kind` for non-raising lookups in hot paths.
109 _VALID_KIND_VALUES: Final[frozenset[str]] = frozenset(k.value for k in MemoryEventKind)
110
111 #: Allowed values for :attr:`MemoryEventRecord.status`. Mirrors
112 #: ``MEMORY_EVENT_STATUSES`` in ``memory-event.mjs``.
113 _VALID_STATUSES: Final[frozenset[str]] = frozenset({"success", "failed"})
114
115
116 # ──────────────────────────────────────────────────────────────────────────────
117 # Validation primitives
118 # ──────────────────────────────────────────────────────────────────────────────
119
120 #: Event-id format: literal ``mem_`` prefix + exactly 12 lowercase hex chars.
121 #: Matches the output of ``crypto.randomBytes(6).toString('hex')`` in JS.
122 _EVENT_ID_RE: Final[re.Pattern[str]] = re.compile(r"^mem_[0-9a-f]{12}$")
123
124 #: Sensitive-key pattern. Mirrors the regex literal in ``memory-event.mjs``::
125 #:
126 #: /(api[_-]?key|secret|password|token|credential|authorization|bearer|private[_-]?key)/i
127 #:
128 #: Note: this uses :func:`re.search` (substring match), not :func:`re.fullmatch`.
129 #: A key like ``"my_api_key_for_x"`` is therefore rejected, exactly as the JS
130 #: side rejects it.
131 _SENSITIVE_VALUE_RE: Final[re.Pattern[str]] = re.compile(
132 r"(api[_-]?key|secret|password|token|credential|authorization|bearer|private[_-]?key)",
133 re.IGNORECASE,
134 )
135
136 #: Maximum recursion depth for the sensitive-key scan. Matches the
137 #: ``depth > 8`` short-circuit in JS ``hasSensitiveKeys``.
138 _MAX_SENSITIVE_SCAN_DEPTH: Final[int] = 8
139
140
141 def _has_sensitive_keys(obj: Any, depth: int = 0) -> bool:
142 """Return ``True`` if *obj* contains a sensitive key within depth bounds.
143
144 Mirrors :func:`hasSensitiveKeys` in :file:`knowtation/lib/memory-event.mjs`.
145 The traversal walks dicts and lists; scalars are skipped (only *keys* are
146 checked, not values, matching the JS implementation). When ``depth`` is
147 strictly greater than :data:`_MAX_SENSITIVE_SCAN_DEPTH` the function
148 returns ``False`` immediately — this is a deliberate amplification-attack
149 safeguard, not a correctness defect, and is observable from the test
150 suite (Tier 7 security tests).
151
152 Args:
153 obj: Arbitrary value to scan; only ``dict`` and ``list`` are recursed.
154 depth: Current recursion depth. Callers should leave at the default 0.
155
156 Returns:
157 ``True`` if any key (case-insensitive) at depth ≤ 8 matches the
158 sensitive pattern; ``False`` otherwise.
159 """
160 if depth > _MAX_SENSITIVE_SCAN_DEPTH:
161 return False
162 if obj is None:
163 return False
164 if isinstance(obj, dict):
165 for k, v in obj.items():
166 if isinstance(k, str) and _SENSITIVE_VALUE_RE.search(k):
167 return True
168 if isinstance(v, (dict, list)) and _has_sensitive_keys(v, depth + 1):
169 return True
170 return False
171 if isinstance(obj, list):
172 for item in obj:
173 if isinstance(item, (dict, list)) and _has_sensitive_keys(item, depth + 1):
174 return True
175 return False
176 return False
177
178
179 def is_valid_event_kind(kind: str) -> bool:
180 """Return ``True`` iff *kind* is one of the 15 canonical event kinds.
181
182 Non-raising predicate suitable for hot-path validation (e.g. while
183 streaming events from disk). Strict equality on the string value;
184 no aliasing or case folding.
185
186 Args:
187 kind: Candidate string to test.
188
189 Returns:
190 ``True`` iff *kind* is a member of :class:`MemoryEventKind`'s value set.
191
192 Examples:
193 >>> is_valid_event_kind("search")
194 True
195 >>> is_valid_event_kind("recall") # historical name in the issue text
196 False
197 >>> is_valid_event_kind(123) # type: ignore[arg-type]
198 False
199 """
200 return isinstance(kind, str) and kind in _VALID_KIND_VALUES
201
202
203 # ──────────────────────────────────────────────────────────────────────────────
204 # Event record — frozen dataclass
205 # ──────────────────────────────────────────────────────────────────────────────
206
207
208 @dataclass(frozen=True, slots=True)
209 class MemoryEventRecord:
210 """One validated memory event as it appears in Muse commit metadata.
211
212 Constructing an instance runs every validation rule (see
213 :meth:`__post_init__`); a successfully constructed record is therefore
214 guaranteed wire-safe. The dataclass is :py:obj:`frozen` so records
215 cannot be mutated after creation — any field change must reconstruct
216 the record with :func:`dataclasses.replace`, which re-runs validation.
217
218 Fields:
219 event_id: ``"mem_" + 12 lowercase hex chars``. Matches
220 :data:`_EVENT_ID_RE`.
221 kind: One of the 15 :class:`MemoryEventKind` members.
222 ts: ISO-8601 timestamp string parseable by
223 :meth:`datetime.datetime.fromisoformat`.
224 vault_id: Non-empty vault identifier.
225 agent_id: Optional ``--agent-id`` value passed to ``muse commit``;
226 may be an opaque string today and an identity-domain handle in
227 Phase 7.7.
228 model_id: Optional ``--model-id`` value passed to ``muse commit``.
229 status: ``"success"`` (default) or ``"failed"``.
230 ttl: Optional time-to-live hint (free-form string; consolidator
231 interprets it).
232 air_id: Optional ICP/AIR attestation receipt identifier added by
233 Phase 7.x.
234 data: Arbitrary JSON-serialisable payload. **Must not** contain any
235 sensitive keys (recursive scan, depth ≤ 8).
236
237 Raises:
238 ValueError: When any field violates the schema. See
239 :meth:`__post_init__` for the full list of rules.
240 """
241
242 event_id: str
243 kind: MemoryEventKind
244 ts: str
245 vault_id: str
246 agent_id: str | None = None
247 model_id: str | None = None
248 status: Literal["success", "failed"] = "success"
249 ttl: str | None = None
250 air_id: str | None = None
251 data: dict[str, Any] = field(default_factory=dict)
252
253 def __post_init__(self) -> None:
254 """Validate every field; raise :class:`ValueError` on any violation.
255
256 The Python type system enforces some of these constraints at static
257 check time, but the dataclass constructor accepts arbitrary inputs at
258 runtime (e.g. ``MemoryEventRecord(**json_dict)``), so all checks are
259 repeated here.
260
261 Order of checks is shallow→deep so the cheapest rejections happen
262 first. This matters for the Tier 7 security tests where we feed in
263 adversarial 10 000-character ``kind`` strings.
264 """
265 if not isinstance(self.event_id, str) or not _EVENT_ID_RE.match(self.event_id):
266 raise ValueError(
267 f"event_id must match ^mem_[0-9a-f]{{12}}$, got {self.event_id!r}"
268 )
269
270 if not isinstance(self.kind, MemoryEventKind):
271 raise ValueError(
272 f"kind must be a MemoryEventKind, got {type(self.kind).__name__}"
273 )
274 if self.kind.value not in _VALID_KIND_VALUES:
275 raise ValueError(f"unknown MemoryEventKind value: {self.kind.value!r}")
276
277 if not isinstance(self.ts, str):
278 raise ValueError(f"ts must be a string, got {type(self.ts).__name__}")
279 try:
280 datetime.fromisoformat(self.ts)
281 except ValueError as exc:
282 raise ValueError(
283 f"ts must be parseable by datetime.fromisoformat: {exc}"
284 ) from None
285
286 if not isinstance(self.vault_id, str) or not self.vault_id:
287 raise ValueError("vault_id must be a non-empty string")
288
289 if self.status not in _VALID_STATUSES:
290 raise ValueError(
291 f"status must be one of {sorted(_VALID_STATUSES)!r}, got {self.status!r}"
292 )
293
294 for opt_name in ("agent_id", "model_id", "ttl", "air_id"):
295 opt_val = getattr(self, opt_name)
296 if opt_val is not None and not isinstance(opt_val, str):
297 raise ValueError(
298 f"{opt_name} must be str or None, got {type(opt_val).__name__}"
299 )
300
301 if not isinstance(self.data, dict):
302 raise ValueError(
303 f"data must be a dict, got {type(self.data).__name__}"
304 )
305 if _has_sensitive_keys(self.data):
306 raise ValueError(
307 "data contains sensitive key patterns "
308 "(api_key, secret, password, token, credential, authorization, "
309 "bearer, private_key); remove secrets before constructing the event"
310 )
311
312
313 # ──────────────────────────────────────────────────────────────────────────────
314 # Serialisation
315 # ──────────────────────────────────────────────────────────────────────────────
316
317
318 def from_dict(d: dict[str, Any]) -> MemoryEventRecord:
319 """Deserialise a plain dict into a :class:`MemoryEventRecord`.
320
321 Required keys: ``event_id``, ``kind``, ``ts``, ``vault_id``.
322 Optional keys: ``agent_id``, ``model_id``, ``status``, ``ttl``, ``air_id``,
323 ``data``. Any other key is silently ignored for forward-compatibility
324 with future schema additions — callers that care about strict validation
325 should compare ``set(d.keys())`` against the expected schema themselves.
326
327 Args:
328 d: Dict matching the canonical Python field shape.
329
330 Returns:
331 Validated :class:`MemoryEventRecord`.
332
333 Raises:
334 ValueError: For any of the following:
335
336 * *d* is not a :class:`dict`.
337 * Any required key is missing.
338 * ``kind`` is not a string or is not one of the 15 canonical values.
339 * ``data`` is present and is not a :class:`dict`.
340 * Any constructor-level validation fails (see
341 :meth:`MemoryEventRecord.__post_init__`).
342 """
343 if not isinstance(d, dict):
344 raise ValueError(f"from_dict expects a dict, got {type(d).__name__}")
345
346 required_keys = ("event_id", "kind", "ts", "vault_id")
347 missing = [k for k in required_keys if k not in d]
348 if missing:
349 raise ValueError(f"missing required field(s): {', '.join(missing)}")
350
351 raw_kind = d["kind"]
352 if not isinstance(raw_kind, str):
353 raise ValueError(f"kind must be a string, got {type(raw_kind).__name__}")
354 if not is_valid_event_kind(raw_kind):
355 snippet = raw_kind if len(raw_kind) <= 64 else (raw_kind[:64] + "…")
356 raise ValueError(
357 f"kind {snippet!r} is not one of the {len(_VALID_KIND_VALUES)} "
358 "canonical MemoryEventKind values"
359 )
360 kind = MemoryEventKind(raw_kind)
361
362 raw_data = d.get("data", {})
363 if raw_data is None:
364 raw_data = {}
365 if not isinstance(raw_data, dict):
366 raise ValueError(f"data must be a dict, got {type(raw_data).__name__}")
367
368 return MemoryEventRecord(
369 event_id=d["event_id"],
370 kind=kind,
371 ts=d["ts"],
372 vault_id=d["vault_id"],
373 agent_id=d.get("agent_id"),
374 model_id=d.get("model_id"),
375 status=d.get("status", "success"),
376 ttl=d.get("ttl"),
377 air_id=d.get("air_id"),
378 data=raw_data,
379 )
380
381
382 def to_dict(record: MemoryEventRecord) -> dict[str, Any]:
383 """Serialise a :class:`MemoryEventRecord` into a minimal JSON-friendly dict.
384
385 Optional fields whose value is ``None`` are *omitted* from the output to
386 keep the wire format small and to make round-trip equality unambiguous.
387 The ``kind`` field is emitted as its string value, not the enum object,
388 so the result is JSON-serialisable as-is (``json.dumps(to_dict(r))``).
389
390 The ``data`` payload is deep-copied so callers cannot mutate the
391 record's frozen state through the returned dict. This is the only
392 O(n)-in-data-size operation in serialisation; for the perf budget see
393 :file:`tests/test_knowtation_events.py` Tier 6.
394
395 Args:
396 record: Record to serialise.
397
398 Returns:
399 A new dict containing the canonical wire-format keys.
400 """
401 out: dict[str, Any] = {
402 "event_id": record.event_id,
403 "kind": record.kind.value,
404 "ts": record.ts,
405 "vault_id": record.vault_id,
406 "status": record.status,
407 "data": copy.deepcopy(record.data),
408 }
409 if record.agent_id is not None:
410 out["agent_id"] = record.agent_id
411 if record.model_id is not None:
412 out["model_id"] = record.model_id
413 if record.ttl is not None:
414 out["ttl"] = record.ttl
415 if record.air_id is not None:
416 out["air_id"] = record.air_id
417 return out
418
419
420 # ──────────────────────────────────────────────────────────────────────────────
421 # Schema content-address — computed at module load
422 # ──────────────────────────────────────────────────────────────────────────────
423
424
425 def _compute_schema_hash() -> str:
426 """Return SHA-256 hex of the canonical JSON of sorted enum values.
427
428 The output is deterministic given the enum membership and order-stable
429 JSON formatting (sorted list, no whitespace). Callers may compare this
430 against a pinned value to detect schema drift before reading commit
431 metadata they did not write themselves.
432 """
433 sorted_values = sorted(k.value for k in MemoryEventKind)
434 canonical = json.dumps(sorted_values, sort_keys=True, separators=(",", ":"))
435 return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
436
437
438 #: Deterministic SHA-256 hex digest over the sorted list of canonical kind
439 #: values. Stable across Python interpreter restarts and re-imports;
440 #: changes only when :class:`MemoryEventKind`'s membership changes.
441 EVENTS_SCHEMA_HASH: Final[str] = _compute_schema_hash()
442
443
444 __all__ = [
445 "EVENTS_SCHEMA_HASH",
446 "EVENTS_SCHEMA_VERSION",
447 "MemoryEventKind",
448 "MemoryEventRecord",
449 "from_dict",
450 "is_valid_event_kind",
451 "to_dict",
452 ]
File History 2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 26 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 26 days ago