aaronrene / gabriel-muse public
manifest.py python
434 lines 15.0 KB
Raw
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 32 days ago
1 """Hierarchical vault snapshot manifests for the Muse knowtation domain.
2
3 A vault manifest organises a snapshot's notes into a two-level hierarchy::
4
5 ProjectManifest ← top-level project grouping (by frontmatter ``project`` key)
6 └─ NoteEntry ← one Markdown note's metadata + hashes
7
8 This structure enables *partial re-parsing*: when merging, only notes whose
9 ``content_hash`` changed need to be re-parsed by the note engine. Notes in
10 unchanged projects are reused from the cached manifest, making three-way
11 merges on large vaults significantly faster.
12
13 The ``frontmatter_hash`` in :class:`NoteEntry` is a SHA-256 of the note's
14 *frontmatter JSON* rather than its raw bytes. Two notes that differ only in
15 whitespace inside the body but have identical frontmatter will have the same
16 ``frontmatter_hash``.
17
18 Public API
19 ----------
20 - :class:`NoteEntry` — per-note metadata.
21 - :class:`ProjectManifest` — project-level grouping.
22 - :class:`VaultManifest` — complete hierarchical vault manifest.
23 - :func:`build_vault_manifest` — build from a flat snapshot manifest.
24 - :func:`diff_vault_manifests` — find added/removed/modified notes between two.
25 - :func:`write_vault_manifest` — persist to ``.muse/knowtation_manifests/<id>.json``.
26 - :func:`read_vault_manifest` — load from disk.
27 """
28
29 from __future__ import annotations
30
31 import hashlib
32 import json
33 import logging
34 import pathlib
35 from typing import TypedDict
36
37 from muse.core._types import Manifest
38
39 logger = logging.getLogger(__name__)
40
41 type _ProjectNoteMap = dict[str, list[str]] # project → [note_path, ...]
42 type _NoteEntryMap = dict[str, "NoteEntry"] # note_path → NoteEntry
43
44
45 # ---------------------------------------------------------------------------
46 # Data types
47 # ---------------------------------------------------------------------------
48
49
50 class NoteEntry(TypedDict):
51 """Metadata for a single Markdown note.
52
53 ``path`` Workspace-relative POSIX path.
54 ``content_hash`` SHA-256 of the raw note bytes (from the object store).
55 ``frontmatter_hash`` SHA-256 of the note's YAML frontmatter JSON.
56 Empty string when the note has no frontmatter.
57 ``project`` Value of the ``project`` frontmatter key, or ``""``
58 when absent.
59 ``note_type`` File extension classification (``"markdown"`` or
60 ``"other"``).
61 ``section_count`` Number of Markdown headings (H1–H6) in the note.
62 0 for non-Markdown files or parse errors.
63 ``size_bytes`` Raw file size in bytes (0 if unavailable).
64 """
65
66 path: str
67 content_hash: str
68 frontmatter_hash: str
69 project: str
70 note_type: str
71 section_count: int
72 size_bytes: int
73
74
75 class ProjectManifest(TypedDict):
76 """Manifest for one project grouping of notes.
77
78 ``project`` Project name (value of frontmatter ``project`` key,
79 or ``"__root__"`` for notes without a project).
80 ``project_hash`` SHA-256 of all sorted ``content_hash`` values in this
81 project — stable fingerprint for change detection.
82 ``notes`` All :class:`NoteEntry` records in this project.
83 ``total_notes`` Total number of notes tracked in this project.
84 ``markdown_notes`` Number of notes with a ``.md`` extension.
85 """
86
87 project: str
88 project_hash: str
89 notes: list[NoteEntry]
90 total_notes: int
91 markdown_notes: int
92
93
94 class VaultManifest(TypedDict):
95 """Complete hierarchical manifest for one vault snapshot.
96
97 ``snapshot_id`` The snapshot this manifest was built from.
98 ``manifest_hash`` SHA-256 of this manifest's JSON — stable cache key.
99 ``projects`` All :class:`ProjectManifest` entries, sorted by project name.
100 ``total_notes`` Total notes across all projects.
101 ``markdown_notes`` Notes with ``.md`` extension.
102 ``total_sections`` Sum of ``section_count`` across all notes.
103 """
104
105 snapshot_id: str
106 manifest_hash: str
107 projects: list[ProjectManifest]
108 total_notes: int
109 markdown_notes: int
110 total_sections: int
111
112
113 # ---------------------------------------------------------------------------
114 # Note type classification
115 # ---------------------------------------------------------------------------
116
117 _NOTE_SUFFIXES: frozenset[str] = frozenset({".md", ".markdown", ".mdx"})
118
119
120 def _note_type_of(file_path: str) -> str:
121 """Return the note-type label for *file_path* based on its extension."""
122 suffix = pathlib.PurePosixPath(file_path).suffix.lower()
123 return "markdown" if suffix in _NOTE_SUFFIXES else "other"
124
125
126 def _is_markdown(file_path: str) -> bool:
127 """Return ``True`` if *file_path* has a Markdown extension."""
128 suffix = pathlib.PurePosixPath(file_path).suffix.lower()
129 return suffix in _NOTE_SUFFIXES
130
131
132 # ---------------------------------------------------------------------------
133 # Frontmatter and section parsing (lightweight, no external deps)
134 # ---------------------------------------------------------------------------
135
136
137 def _parse_frontmatter_hash(content: bytes) -> tuple[str, str]:
138 """Extract and hash the YAML frontmatter block from *content*.
139
140 Returns a ``(frontmatter_hash, project)`` tuple. When no frontmatter is
141 present, both values are empty strings. Parsing is intentionally minimal
142 at Phase 1.1 — a proper typed ``FrontmatterSchema`` will be introduced in
143 Phase 1.3.
144
145 The hash is computed over the raw YAML text (not parsed values) so that
146 semantically identical frontmatter with different whitespace produces
147 different hashes — acceptable for the manifest cache key until Phase 1.3
148 introduces normalised hashing.
149 """
150 try:
151 text = content.decode("utf-8", errors="replace")
152 except Exception:
153 return "", ""
154
155 if not text.startswith("---"):
156 return "", ""
157
158 # Find the closing "---" marker on its own line.
159 rest = text[3:]
160 end = rest.find("\n---")
161 if end == -1:
162 return "", ""
163
164 raw_fm = rest[:end].strip()
165 fm_hash = hashlib.sha256(raw_fm.encode()).hexdigest()
166
167 # Extract ``project`` value — simple key: value scan (no full YAML parse).
168 project = ""
169 for line in raw_fm.splitlines():
170 stripped = line.strip()
171 if stripped.startswith("project:"):
172 value = stripped[len("project:"):].strip().strip("\"'")
173 project = value
174 break
175
176 return fm_hash, project
177
178
179 def _count_sections(content: bytes) -> int:
180 """Count Markdown headings (# through ######) in *content*.
181
182 Returns 0 when the note has no headings or when *content* cannot be
183 decoded as UTF-8.
184 """
185 try:
186 text = content.decode("utf-8", errors="replace")
187 except Exception:
188 return 0
189
190 # Skip the frontmatter block before counting headings.
191 body = text
192 if text.startswith("---"):
193 rest = text[3:]
194 end = rest.find("\n---")
195 if end != -1:
196 body = rest[end + 4:]
197
198 count = 0
199 for line in body.splitlines():
200 if line.startswith("#"):
201 # Accept # through ###### at the start of a line (ATX headings).
202 stripped = line.lstrip("#")
203 if stripped and (stripped[0] == " " or stripped[0] == "\t"):
204 count += 1
205
206 return count
207
208
209 # ---------------------------------------------------------------------------
210 # Builder
211 # ---------------------------------------------------------------------------
212
213
214 def build_vault_manifest(
215 snapshot_id: str,
216 flat_manifest: Manifest,
217 repo_root: pathlib.Path,
218 ) -> VaultManifest:
219 """Build a :class:`VaultManifest` from a flat ``{path: content_hash}`` dict.
220
221 Attempts to parse frontmatter and count sections for Markdown files.
222 Falls back gracefully for binary files or parse errors. All blob reads
223 go through the Muse object store so only committed content is parsed.
224
225 Args:
226 snapshot_id: The snapshot this manifest represents.
227 flat_manifest: ``{workspace_path: sha256}`` from the snapshot.
228 repo_root: Repository root for object store access.
229
230 Returns:
231 A fully populated :class:`VaultManifest`.
232 """
233 from muse.core.object_store import read_object
234
235 # Group notes by project (frontmatter ``project`` key or ``"__root__"``).
236 entries: list[NoteEntry] = []
237 total_sections = 0
238 total_markdown = 0
239
240 for file_path in sorted(flat_manifest):
241 content_hash = flat_manifest[file_path]
242 note_type = _note_type_of(file_path)
243 fm_hash = ""
244 project = "__root__"
245 section_count = 0
246
247 if note_type == "markdown":
248 source = read_object(repo_root, content_hash)
249 if source is not None:
250 try:
251 fm_hash, raw_project = _parse_frontmatter_hash(source)
252 project = raw_project or "__root__"
253 section_count = _count_sections(source)
254 except Exception:
255 logger.debug("Note parse failed for %s", file_path)
256 total_markdown += 1
257
258 total_sections += section_count
259
260 # size_bytes requires reading the blob; use 0 as a safe default when
261 # the object is not locally cached. Phase 1.4 will hydrate this field.
262 entries.append(NoteEntry(
263 path=file_path,
264 content_hash=content_hash,
265 frontmatter_hash=fm_hash,
266 project=project,
267 note_type=note_type,
268 section_count=section_count,
269 size_bytes=0,
270 ))
271
272 # Group entries by project.
273 project_map: _ProjectNoteMap = {}
274 entry_map: _NoteEntryMap = {e["path"]: e for e in entries}
275 for entry in entries:
276 project_map.setdefault(entry["project"], []).append(entry["path"])
277
278 projects: list[ProjectManifest] = []
279 for proj_name in sorted(project_map):
280 note_paths = sorted(project_map[proj_name])
281 proj_notes = [entry_map[p] for p in note_paths]
282 proj_hashes = [e["content_hash"] for e in proj_notes]
283 proj_hash = hashlib.sha256(
284 "|".join(sorted(proj_hashes)).encode()
285 ).hexdigest()
286 md_count = sum(1 for n in proj_notes if n["note_type"] == "markdown")
287 projects.append(ProjectManifest(
288 project=proj_name,
289 project_hash=proj_hash,
290 notes=proj_notes,
291 total_notes=len(proj_notes),
292 markdown_notes=md_count,
293 ))
294
295 manifest_json = json.dumps(
296 {"snapshot_id": snapshot_id, "projects": projects}, sort_keys=True
297 )
298 manifest_hash = hashlib.sha256(manifest_json.encode()).hexdigest()
299
300 return VaultManifest(
301 snapshot_id=snapshot_id,
302 manifest_hash=manifest_hash,
303 projects=projects,
304 total_notes=len(flat_manifest),
305 markdown_notes=total_markdown,
306 total_sections=total_sections,
307 )
308
309
310 # ---------------------------------------------------------------------------
311 # Diff
312 # ---------------------------------------------------------------------------
313
314
315 class NoteFileDiff(TypedDict):
316 """Change record from :func:`diff_vault_manifests` for one note."""
317
318 path: str
319 change: str # "added" | "removed" | "modified" | "frontmatter_changed"
320 old_hash: str
321 new_hash: str
322 old_fm_hash: str
323 new_fm_hash: str
324 frontmatter_change: bool # True when fm_hash differs (frontmatter mutated)
325
326
327 def diff_vault_manifests(
328 base: VaultManifest,
329 target: VaultManifest,
330 ) -> list[NoteFileDiff]:
331 """Produce a per-note change list between two :class:`VaultManifest` objects.
332
333 Notes with identical ``content_hash`` values are skipped (no change).
334 Notes where only the ``content_hash`` changed but ``frontmatter_hash`` is
335 the same are marked ``"modified"`` with ``frontmatter_change=False``.
336 Notes where ``frontmatter_hash`` changed are ``"frontmatter_changed"``
337 with ``frontmatter_change=True``.
338
339 Args:
340 base: Manifest for the earlier state.
341 target: Manifest for the later state.
342
343 Returns:
344 Sorted list of :class:`NoteFileDiff` records.
345 """
346 base_notes: _NoteEntryMap = {}
347 for proj in base["projects"]:
348 for note in proj["notes"]:
349 base_notes[note["path"]] = note
350
351 target_notes: _NoteEntryMap = {}
352 for proj in target["projects"]:
353 for note in proj["notes"]:
354 target_notes[note["path"]] = note
355
356 diffs: list[NoteFileDiff] = []
357 all_paths = sorted(set(base_notes) | set(target_notes))
358
359 for path in all_paths:
360 bn = base_notes.get(path)
361 tn = target_notes.get(path)
362
363 if bn is None and tn is not None:
364 diffs.append(NoteFileDiff(
365 path=path, change="added",
366 old_hash="", new_hash=tn["content_hash"],
367 old_fm_hash="", new_fm_hash=tn["frontmatter_hash"],
368 frontmatter_change=True,
369 ))
370 elif bn is not None and tn is None:
371 diffs.append(NoteFileDiff(
372 path=path, change="removed",
373 old_hash=bn["content_hash"], new_hash="",
374 old_fm_hash=bn["frontmatter_hash"], new_fm_hash="",
375 frontmatter_change=True,
376 ))
377 elif bn is not None and tn is not None:
378 if bn["content_hash"] == tn["content_hash"]:
379 continue # No change.
380 fm_changed = bn["frontmatter_hash"] != tn["frontmatter_hash"]
381 diffs.append(NoteFileDiff(
382 path=path,
383 change="frontmatter_changed" if fm_changed else "modified",
384 old_hash=bn["content_hash"], new_hash=tn["content_hash"],
385 old_fm_hash=bn["frontmatter_hash"], new_fm_hash=tn["frontmatter_hash"],
386 frontmatter_change=fm_changed,
387 ))
388
389 return diffs
390
391
392 # ---------------------------------------------------------------------------
393 # Persistence
394 # ---------------------------------------------------------------------------
395
396
397 def write_vault_manifest(
398 repo_root: pathlib.Path, manifest: VaultManifest
399 ) -> None:
400 """Persist a :class:`VaultManifest` to ``.muse/knowtation_manifests/<hash>.json``.
401
402 Creates the directory if absent. Skips writing when the file already
403 exists (content-addressed store is immutable).
404
405 Args:
406 repo_root: Repository root.
407 manifest: The manifest to write.
408 """
409 store_dir = repo_root / ".muse" / "knowtation_manifests"
410 store_dir.mkdir(parents=True, exist_ok=True)
411 path = store_dir / f"{manifest['manifest_hash']}.json"
412 if not path.exists():
413 path.write_text(json.dumps(manifest), encoding="utf-8")
414
415
416 def read_vault_manifest(
417 repo_root: pathlib.Path, manifest_hash: str
418 ) -> VaultManifest | None:
419 """Load a :class:`VaultManifest` by its hash.
420
421 Args:
422 repo_root: Repository root.
423 manifest_hash: The ``manifest_hash`` of the target manifest.
424
425 Returns:
426 The deserialized :class:`VaultManifest`, or ``None`` if not found.
427 """
428 path = (
429 repo_root / ".muse" / "knowtation_manifests" / f"{manifest_hash}.json"
430 )
431 if not path.exists():
432 return None
433 raw: VaultManifest = json.loads(path.read_text(encoding="utf-8"))
434 return raw
File History 2 commits
sha256:660fcac1df3ab28f61862e961890bd2ca8b754fa0242079d93ca1e25037ec8a6 chore(tests): add docstring to tests/__init__.py so rc14 tr… Human 32 days ago
sha256:d8316ffae901be06347e16ab55be11868eb519dd16ade3e8aa16a99e662f7e62 baseline: rc14 re-baseline after rc3 store corruption recovery Human patch 32 days ago