aaronrene / overseer-kit public
muse_artifact.py python
160 lines 6.8 KB
Raw
sha256:ff22b6a82a08c117176c3e2603264a0e49fe8204ce57c3cefe7f5b165a8634f8 docs: recenter public story on governance loop Human 7 days ago
1 """Reproduce or verify the explicitly local Muse rc5 recovery distribution.
2
3 No upstream release is claimed. This stdlib-only tool neither installs nor
4 downloads anything. The reviewed manifest covers source, data and metadata.
5 """
6 import argparse
7 import base64
8 import csv
9 import hashlib
10 import io
11 import json
12 from pathlib import Path, PurePosixPath
13 import stat
14 import zipfile
15
16 HERE = Path(__file__).resolve().parent
17 DIST = 'muse-0.2.1rc5.dist-info'
18 SOURCE_NAME = 'muse-0.2.1rc5-overseer-recovery1-source.zip'
19 WHEEL_NAME = 'muse-0.2.1rc5-1overseerrecovery-py3-none-any.whl'
20
21
22 def sha(raw):
23 return hashlib.sha256(raw).hexdigest()
24
25
26 def json_bytes(value):
27 return (json.dumps(value, indent=2, sort_keys=True) + '\n').encode()
28
29
30 def contract():
31 lock = json.loads((HERE / 'muse-rc5-artifact.json').read_bytes())
32 raw = (HERE / 'muse-rc5-source-files.json').read_bytes()
33 if sha(raw) != lock['payload_manifest_sha256']:
34 raise ValueError('reviewed payload manifest digest differs')
35 return lock, json.loads(raw)
36
37
38 def validate_payload(files, manifest, source_sha256):
39 if files.keys() != manifest.keys():
40 raise ValueError('payload paths differ from reviewed manifest')
41 source = hashlib.sha256()
42 for name, expected in sorted(manifest.items()):
43 path = PurePosixPath(name)
44 if (path.is_absolute() or '..' in path.parts or str(path) != name
45 or not name.startswith(('muse/', DIST + '/'))):
46 raise ValueError('invalid payload path: ' + name)
47 raw = files[name]
48 if len(raw) != expected['size'] or sha(raw) != expected['sha256']:
49 raise ValueError('payload differs: ' + name)
50 if name.startswith('muse/') and name.endswith('.py'):
51 source.update(name[5:].encode() + b'\0' + raw + b'\0')
52 if source.hexdigest() != source_sha256:
53 raise ValueError('package source differs from reviewed rc5')
54
55
56 def read_installed(root, manifest):
57 files = {}
58 for name in manifest:
59 path = root / name
60 if any(p.is_symlink() for p in [path, *path.parents]):
61 raise ValueError('linked input: ' + name)
62 files[name] = path.read_bytes()
63 # Installation-generated metadata/bytecode are not distribution inputs.
64 # Do not silently omit added package code or data.
65 observed = {p.relative_to(root).as_posix() for p in (root / 'muse').rglob('*')
66 if p.is_file() and p.suffix != '.pyc'}
67 if observed != {n for n in manifest if n.startswith('muse/')}:
68 raise ValueError('installed package has missing or extra payload files')
69 return files
70
71
72 def read_archive(raw, manifest):
73 with zipfile.ZipFile(io.BytesIO(raw)) as archive:
74 names = archive.namelist()
75 if len(names) != len(set(names)) or set(names) != set(manifest):
76 raise ValueError('source archive paths differ from reviewed manifest')
77 for info in archive.infolist():
78 if info.file_size != manifest[info.filename]['size']:
79 raise ValueError('source archive entry size differs')
80 return {name: archive.read(name) for name in names}
81
82
83 def archive_bytes(files):
84 # Stored entries avoid zlib/version-dependent compression; metadata/order
85 # never depend on wall clock, filesystem mode, umask, host or source mtime.
86 output = io.BytesIO()
87 with zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_STORED) as archive:
88 for name, raw in sorted(files.items()):
89 info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
90 info.create_system = 3
91 info.external_attr = (stat.S_IFREG | 0o644) << 16
92 archive.writestr(info, raw)
93 return output.getvalue()
94
95
96 def artifacts(files, lock):
97 source = archive_bytes(files)
98 wheel_files = dict(files)
99 wheel_files[DIST + '/WHEEL'] = (
100 'Wheel-Version: 1.0\nGenerator: overseer-rc5-recovery-1\n'
101 'Root-Is-Purelib: true\nBuild: 1overseerrecovery\nTag: py3-none-any\n'
102 ).encode()
103 wheel_files[DIST + '/OVERSEER_RECOVERY.json'] = json_bytes({
104 'origin': lock['origin'], 'upstream_release': False,
105 'version': '0.2.1rc5', 'build': '1overseerrecovery',
106 'source_sha256': lock['package_source_sha256'],
107 'payload_manifest_sha256': lock['payload_manifest_sha256'],
108 'recovery_source_archive_sha256': sha(source),
109 'original_sdist_sha256_recorded_by_pip': lock['original_sdist_sha256'],
110 'original_sdist_recovered': False,
111 })
112 record = io.StringIO()
113 writer = csv.writer(record, lineterminator='\n')
114 for name, raw in sorted(wheel_files.items()):
115 digest = base64.urlsafe_b64encode(hashlib.sha256(raw).digest()).decode().rstrip('=')
116 writer.writerow([name, 'sha256=' + digest, len(raw)])
117 writer.writerow([DIST + '/RECORD', '', ''])
118 wheel_files[DIST + '/RECORD'] = record.getvalue().encode()
119 return {SOURCE_NAME: source, WHEEL_NAME: archive_bytes(wheel_files)}
120
121
122 def main():
123 parser = argparse.ArgumentParser(description=__doc__)
124 commands = parser.add_subparsers(dest='command', required=True)
125 rebuild = commands.add_parser('rebuild')
126 inputs = rebuild.add_mutually_exclusive_group(required=True)
127 inputs.add_argument('--installed-root', type=Path)
128 inputs.add_argument('--source-archive', type=Path)
129 rebuild.add_argument('--output-dir', type=Path, required=True)
130 verify = commands.add_parser('verify')
131 verify.add_argument('--wheel', type=Path, required=True)
132 args = parser.parse_args()
133 try:
134 lock, manifest = contract()
135 if args.command == 'verify':
136 if args.wheel.name != WHEEL_NAME or sha(args.wheel.read_bytes()) != lock['artifacts'][WHEEL_NAME]:
137 raise ValueError('wheel name/digest differs from reviewed recovery artifact')
138 print('Verified exact local rc5 recovery wheel; not an upstream release.')
139 return
140 if args.installed_root:
141 files = read_installed(args.installed_root.resolve(), manifest)
142 else:
143 raw = args.source_archive.read_bytes()
144 if sha(raw) != lock['artifacts'][SOURCE_NAME]:
145 raise ValueError('recovery source archive digest differs')
146 files = read_archive(raw, manifest)
147 validate_payload(files, manifest, lock['package_source_sha256'])
148 built = artifacts(files, lock)
149 if {name: sha(raw) for name, raw in built.items()} != lock['artifacts']:
150 raise ValueError('rebuilt artifact digests differ from reviewed contract')
151 args.output_dir.mkdir(parents=True, exist_ok=False)
152 for name, raw in built.items():
153 (args.output_dir / name).write_bytes(raw)
154 print(json.dumps({name: sha(raw) for name, raw in built.items()}, indent=2))
155 except (ValueError, OSError, zipfile.BadZipFile) as error:
156 parser.exit(1, str(error) + '\n')
157
158
159 if __name__ == '__main__':
160 main()
File History 1 commit
sha256:ff22b6a82a08c117176c3e2603264a0e49fe8204ce57c3cefe7f5b165a8634f8 docs: recenter public story on governance loop Human 7 days ago