install.sh file-level

at sha256:3 · View file ↗ · Intel ↗

History
1 files
1 commits
0 hotspots
0 🧊 dead
0 💥 blast risk
sha256:4 fix(security): pin patched transitive deps to clear Dependabot moderate… · aaronrene · Jun 11, 2026
1 #!/usr/bin/env bash
2 ######################################################################
3 # Paperclip orchestrator — one-shot install on Ubuntu 24.04 LTS
4 #
5 # Run AS ROOT (or via `sudo bash install.sh`) on a fresh AWS t3.medium
6 # provisioned by deploy/paperclip/terraform.
7 #
8 # Idempotent: safe to re-run. Skips steps already complete.
9 #
10 # What this does:
11 # 1. Creates 'paperclip' service user (locked-down, no shell)
12 # 2. Installs Node 20 LTS via NodeSource
13 # 3. Installs pnpm
14 # 4. Installs PostgreSQL 16 + creates database 'paperclip' + role 'paperclip'
15 # 5. Clones Paperclip from PAPERCLIP_REPO_URL into /opt/paperclip
16 # 6. Builds all workspace packages (tsc → dist/) + patches subpath exports
17 # 7. Runs Paperclip migrations
18 # 8. Installs nginx + certbot (Let's Encrypt)
19 # 9. Configures nginx reverse proxy (HTTP-only by default; you add HTTPS via certbot if desired)
20 # 10. Installs systemd service 'paperclip.service' (runs compiled JS, no tsx runtime)
21 # 11. Loads SSM secrets into the systemd EnvironmentFile every 60 seconds via 'paperclip-secrets-sync.timer'
22 #
23 # What this does NOT do:
24 # - Push secrets (run scripts/push-secrets.sh after this completes)
25 # - Wire Knowtation MCP (run scripts/wire-knowtation-mcp.sh after secrets)
26 # - Load skills/agents (run scripts/load-skills-and-agents.sh after MCP)
27 ######################################################################
28
29 set -euo pipefail
30
31 ############
32 # Settings
33 ############
34
35 # Paperclip source — public upstream (default branch is master).
36 # Override with forks via: PAPERCLIP_REPO_URL=... PAPERCLIP_REPO_REF=... when invoking install.sh
37 : "${PAPERCLIP_REPO_URL:=https://github.com/paperclipai/paperclip.git}"
38 : "${PAPERCLIP_REPO_REF:=master}"
39 : "${PAPERCLIP_INSTALL_DIR:=/opt/paperclip}"
40 : "${PAPERCLIP_USER:=paperclip}"
41 : "${PAPERCLIP_DB:=paperclip}"
42 : "${PAPERCLIP_PORT:=3000}"
43 : "${SSM_NAMESPACE:=/knowtation/paperclip/}"
44 # When install is run as `curl … | bash`, $0 is bash — operator assets are fetched from this raw URL instead.
45 : "${KNOWTATION_RAW_DEPLOY_BASE:=https://raw.githubusercontent.com/aaronrene/knowtation/main/deploy/paperclip}"
46
47 LOG_FILE=/var/log/paperclip-install.log
48 exec > >(tee -a "$LOG_FILE") 2>&1
49 echo "[$(date -u +%FT%TZ)] install.sh starting"
50
51 if [[ "$(id -u)" -ne 0 ]]; then
52 echo "ERROR: install.sh must run as root. Use 'sudo bash install.sh'."
53 exit 1
54 fi
55
56 if ! command -v aws &>/dev/null; then
57 echo "ERROR: AWS CLI not installed. Did user-data run? Re-install with:"
58 echo " curl -fsSL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o /tmp/awscliv2.zip && unzip -q /tmp/awscliv2.zip -d /tmp && /tmp/aws/install"
59 exit 1
60 fi
61
62 REGION=$(curl -fsSL -H "X-aws-ec2-metadata-token: $(curl -fsSL -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 60')" 'http://169.254.169.254/latest/meta-data/placement/region' || echo 'us-west-2')
63 echo "[install] AWS region detected: $REGION"
64
65 #####################
66 # 1. Service user
67 #####################
68
69 if ! id -u "$PAPERCLIP_USER" &>/dev/null; then
70 echo "[install] Creating service user: $PAPERCLIP_USER"
71 useradd --system --home-dir "$PAPERCLIP_INSTALL_DIR" --shell /usr/sbin/nologin "$PAPERCLIP_USER"
72 else
73 echo "[install] Service user $PAPERCLIP_USER already exists, skipping"
74 fi
75
76 #####################
77 # 2. Node 20 LTS
78 #####################
79
80 if ! node -v 2>/dev/null | grep -q '^v20'; then
81 echo "[install] Installing Node 20 LTS via NodeSource"
82 curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
83 apt-get install -y nodejs
84 else
85 echo "[install] Node 20 already installed: $(node -v)"
86 fi
87
88 #####################
89 # 3. pnpm
90 #####################
91
92 if ! command -v pnpm &>/dev/null; then
93 echo "[install] Installing pnpm via corepack"
94 corepack enable
95 corepack prepare pnpm@latest --activate
96 else
97 echo "[install] pnpm already installed: $(pnpm -v)"
98 fi
99
100 #####################
101 # 4. PostgreSQL 16
102 #####################
103
104 if ! command -v psql &>/dev/null; then
105 echo "[install] Installing PostgreSQL 16"
106 install -d /usr/share/postgresql-common/pgdg
107 curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/postgresql-common/pgdg/pgdg.gpg
108 echo "deb [signed-by=/usr/share/postgresql-common/pgdg/pgdg.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
109 apt-get update
110 apt-get install -y postgresql-16 postgresql-contrib-16
111 systemctl enable --now postgresql
112 else
113 echo "[install] PostgreSQL already installed: $(psql --version)"
114 fi
115
116 # Create database + role (idempotent).
117 sudo -u postgres psql -tc "SELECT 1 FROM pg_roles WHERE rolname = '$PAPERCLIP_USER'" | grep -q 1 || \
118 sudo -u postgres psql -c "CREATE ROLE $PAPERCLIP_USER WITH LOGIN PASSWORD '$(openssl rand -hex 16)';"
119
120 sudo -u postgres psql -tc "SELECT 1 FROM pg_database WHERE datname = '$PAPERCLIP_DB'" | grep -q 1 || \
121 sudo -u postgres psql -c "CREATE DATABASE $PAPERCLIP_DB OWNER $PAPERCLIP_USER;"
122
123 #####################
124 # 5. Clone Paperclip
125 #####################
126
127 if [[ ! -d "$PAPERCLIP_INSTALL_DIR/.git" ]]; then
128 echo "[install] Cloning Paperclip from $PAPERCLIP_REPO_URL@$PAPERCLIP_REPO_REF"
129 rm -rf "$PAPERCLIP_INSTALL_DIR"
130 git clone --branch "$PAPERCLIP_REPO_REF" "$PAPERCLIP_REPO_URL" "$PAPERCLIP_INSTALL_DIR"
131 else
132 echo "[install] Paperclip already cloned, fetching latest on $PAPERCLIP_REPO_REF"
133 cd "$PAPERCLIP_INSTALL_DIR"
134 git fetch origin "$PAPERCLIP_REPO_REF"
135 git checkout "$PAPERCLIP_REPO_REF"
136 git pull --ff-only origin "$PAPERCLIP_REPO_REF"
137 fi
138
139 # Mirror Knowtation deploy artifacts into /opt/paperclip (scripts, skills, agents).
140 # Piped installs (`curl | bash`): $0 is bash — local DEPLOY_DIR is wrong; fetch from KNOWTATION_RAW_DEPLOY_BASE.
141 mkdir -p "$PAPERCLIP_INSTALL_DIR/scripts" "$PAPERCLIP_INSTALL_DIR/skills" "$PAPERCLIP_INSTALL_DIR/agents"
142 DEPLOY_DIR=""
143 if [[ -n "${BASH_SOURCE[0]:-}" && "${BASH_SOURCE[0]}" != bash && "${BASH_SOURCE[0]}" != */bash ]]; then
144 DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
145 elif [[ -f "$(readlink -f "$0" 2>/dev/null || true)" ]]; then
146 DEPLOY_DIR="$(cd "$(dirname "$(readlink -f "$0")")" && pwd)"
147 fi
148 if [[ -n "$DEPLOY_DIR" && -f "$DEPLOY_DIR/scripts/push-secrets.sh" ]]; then
149 echo "[install] Copying operator scripts from $DEPLOY_DIR/scripts"
150 cp -f "$DEPLOY_DIR"/scripts/*.sh "$PAPERCLIP_INSTALL_DIR/scripts/"
151 else
152 echo "[install] Fetching operator scripts from $KNOWTATION_RAW_DEPLOY_BASE/scripts/"
153 for f in push-secrets.sh hello-world-test.sh wire-knowtation-mcp.sh load-skills-and-agents.sh run-controller.sh; do
154 curl -fsSL "$KNOWTATION_RAW_DEPLOY_BASE/scripts/$f" -o "$PAPERCLIP_INSTALL_DIR/scripts/$f"
155 done
156 fi
157 chmod +x "$PAPERCLIP_INSTALL_DIR/scripts/"*.sh 2>/dev/null || true
158
159 if [[ -n "$DEPLOY_DIR" && -d "$DEPLOY_DIR/skills" ]]; then
160 cp -rf "$DEPLOY_DIR"/skills/* "$PAPERCLIP_INSTALL_DIR/skills/" 2>/dev/null || true
161 cp -rf "$DEPLOY_DIR"/agents/* "$PAPERCLIP_INSTALL_DIR/agents/" 2>/dev/null || true
162 else
163 echo "[install] Fetching skills from $KNOWTATION_RAW_DEPLOY_BASE/skills/"
164 SKILL_FILES=(
165 write-draft.mjs search-vault.mjs read-style-guide.mjs read-positioning.mjs read-playbook.mjs
166 hub-client.mjs heygen-render.mjs elevenlabs-tts.mjs descript-import.mjs
167 )
168 for f in "${SKILL_FILES[@]}"; do
169 curl -fsSL "$KNOWTATION_RAW_DEPLOY_BASE/skills/$f" -o "$PAPERCLIP_INSTALL_DIR/skills/$f"
170 done
171 echo "[install] Fetching agents from $KNOWTATION_RAW_DEPLOY_BASE/agents/"
172 AGENT_FILES=(
173 knowtation/project.yaml bornfree/project.yaml storefree/project.yaml controller/controller.yaml
174 _universal-preamble.yaml
175 _templates/script-writer.yaml _templates/blog-seo.yaml _templates/newsletter.yaml
176 _templates/social-poster.yaml _templates/clip-factory.yaml _templates/thumbnail-brief.yaml
177 )
178 for f in "${AGENT_FILES[@]}"; do
179 mkdir -p "$PAPERCLIP_INSTALL_DIR/agents/$(dirname "$f")"
180 curl -fsSL "$KNOWTATION_RAW_DEPLOY_BASE/agents/$f" -o "$PAPERCLIP_INSTALL_DIR/agents/$f"
181 done
182 fi
183
184 chown -R "$PAPERCLIP_USER:$PAPERCLIP_USER" "$PAPERCLIP_INSTALL_DIR"
185
186 #####################
187 # 6. Install + migrate Paperclip
188 #####################
189
190 echo "[install] Running pnpm install (includes devDeps needed for build)"
191 sudo -u "$PAPERCLIP_USER" -H bash -c "cd $PAPERCLIP_INSTALL_DIR && pnpm install --frozen-lockfile"
192
193 echo "[install] Building workspace packages (compiling TypeScript → JavaScript)"
194 sudo -u "$PAPERCLIP_USER" -H bash -c "cd $PAPERCLIP_INSTALL_DIR && NODE_OPTIONS='--max-old-space-size=2048' pnpm -r build"
195
196 echo "[install] Patching package.json wildcard exports to use compiled dist/ paths"
197 python3 -c "
198 import json, glob
199 pkgs = glob.glob('$PAPERCLIP_INSTALL_DIR/packages/*/package.json')
200 for path in pkgs:
201 with open(path) as f:
202 pkg = json.load(f)
203 exports = pkg.get('exports', {})
204 changed = False
205 for key in list(exports.keys()):
206 val = exports[key]
207 if isinstance(val, str) and val.startswith('./src/') and val.endswith('.ts'):
208 exports[key] = val.replace('./src/', './dist/').replace('.ts', '.js')
209 changed = True
210 if changed:
211 with open(path, 'w') as f:
212 json.dump(pkg, f, indent=2)
213 print('[install] Patched exports:', path)
214 print('[install] Export patch complete')
215 "
216
217 echo "[install] Running database migrations (if Paperclip ships them)"
218 if [[ -f "$PAPERCLIP_INSTALL_DIR/package.json" ]] && grep -q '"migrate"' "$PAPERCLIP_INSTALL_DIR/package.json"; then
219 sudo -u "$PAPERCLIP_USER" -H bash -c "cd $PAPERCLIP_INSTALL_DIR && DATABASE_URL=postgresql://$PAPERCLIP_USER@localhost/$PAPERCLIP_DB pnpm migrate"
220 else
221 echo "[install] No 'migrate' script in package.json — skipping. Paperclip may auto-migrate on first run."
222 fi
223
224 #####################
225 # 7. nginx + certbot
226 #####################
227
228 if ! command -v nginx &>/dev/null; then
229 echo "[install] Installing nginx + certbot"
230 apt-get install -y nginx certbot python3-certbot-nginx
231 fi
232
233 # Default nginx site: reverse-proxy to Paperclip on $PAPERCLIP_PORT, HTTP only.
234 # HTTPS is opt-in via 'sudo certbot --nginx -d <your-domain>' AFTER you point DNS.
235 cat > /etc/nginx/sites-available/paperclip <<EOF
236 server {
237 listen 80;
238 server_name _;
239
240 # Tailscale internal access. Allow from 100.64.0.0/10 (Tailscale CGNAT range) + localhost only.
241 allow 100.64.0.0/10;
242 allow 127.0.0.1;
243 deny all;
244
245 location / {
246 proxy_pass http://127.0.0.1:$PAPERCLIP_PORT;
247 proxy_set_header Host \$host;
248 proxy_set_header X-Real-IP \$remote_addr;
249 proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
250 proxy_set_header X-Forwarded-Proto \$scheme;
251 proxy_http_version 1.1;
252 proxy_set_header Upgrade \$http_upgrade;
253 proxy_set_header Connection "upgrade";
254 }
255 }
256 EOF
257
258 ln -sf /etc/nginx/sites-available/paperclip /etc/nginx/sites-enabled/paperclip
259 rm -f /etc/nginx/sites-enabled/default
260 nginx -t
261 systemctl reload nginx
262
263 #####################
264 # 8. SSM secrets sync
265 #####################
266
267 # Pull ALL parameters under $SSM_NAMESPACE into /etc/paperclip/env.
268 # Re-runs every 60 seconds via paperclip-secrets-sync.timer so JWT rotation is hot.
269
270 mkdir -p /etc/paperclip
271 chown root:"$PAPERCLIP_USER" /etc/paperclip
272 chmod 750 /etc/paperclip
273
274 cat > /usr/local/bin/paperclip-secrets-sync <<'EOF'
275 #!/usr/bin/env bash
276 set -euo pipefail
277
278 SSM_NAMESPACE="${SSM_NAMESPACE:-/knowtation/paperclip/}"
279 ENV_FILE=/etc/paperclip/env
280 TMP_FILE=$(mktemp)
281
282 REGION=$(curl -fsSL -H "X-aws-ec2-metadata-token: $(curl -fsSL -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 60')" 'http://169.254.169.254/latest/meta-data/placement/region')
283
284 aws ssm get-parameters-by-path \
285 --path "$SSM_NAMESPACE" \
286 --recursive \
287 --with-decryption \
288 --region "$REGION" \
289 --output json | jq -r '.Parameters[] | "\(.Name | split("/") | last)=\(.Value)"' > "$TMP_FILE"
290
291 # Atomic replace.
292 chmod 640 "$TMP_FILE"
293 chown root:paperclip "$TMP_FILE"
294 mv "$TMP_FILE" "$ENV_FILE"
295
296 # If anything changed, restart Paperclip. Compare via checksum.
297 CHECKSUM_FILE=/var/lib/paperclip-secrets.checksum
298 NEW_SUM=$(sha256sum "$ENV_FILE" | cut -d' ' -f1)
299 OLD_SUM=$(cat "$CHECKSUM_FILE" 2>/dev/null || echo "")
300
301 if [[ "$NEW_SUM" != "$OLD_SUM" ]]; then
302 echo "$NEW_SUM" > "$CHECKSUM_FILE"
303 systemctl is-active --quiet paperclip.service && systemctl reload-or-restart paperclip.service || true
304 fi
305 EOF
306 chmod +x /usr/local/bin/paperclip-secrets-sync
307
308 cat > /etc/systemd/system/paperclip-secrets-sync.service <<EOF
309 [Unit]
310 Description=Pull Paperclip secrets from AWS SSM Parameter Store into /etc/paperclip/env
311 After=network-online.target
312 Wants=network-online.target
313
314 [Service]
315 Type=oneshot
316 Environment=SSM_NAMESPACE=$SSM_NAMESPACE
317 ExecStart=/usr/local/bin/paperclip-secrets-sync
318 EOF
319
320 cat > /etc/systemd/system/paperclip-secrets-sync.timer <<EOF
321 [Unit]
322 Description=Sync Paperclip secrets every 60 seconds
323
324 [Timer]
325 OnBootSec=30s
326 OnUnitActiveSec=60s
327 Unit=paperclip-secrets-sync.service
328
329 [Install]
330 WantedBy=timers.target
331 EOF
332
333 #####################
334 # 9. systemd unit
335 #####################
336
337 cat > /etc/systemd/system/paperclip.service <<EOF
338 [Unit]
339 Description=Paperclip orchestrator (Knowtation video factory)
340 After=network.target postgresql.service paperclip-secrets-sync.service
341 Wants=postgresql.service paperclip-secrets-sync.service
342
343 [Service]
344 Type=simple
345 User=$PAPERCLIP_USER
346 Group=$PAPERCLIP_USER
347 WorkingDirectory=$PAPERCLIP_INSTALL_DIR
348 EnvironmentFile=/etc/paperclip/env
349 Environment=NODE_ENV=production
350 Environment=PORT=$PAPERCLIP_PORT
351 Environment=DATABASE_URL=postgresql://$PAPERCLIP_USER@localhost/$PAPERCLIP_DB
352 Environment=NODE_OPTIONS=--max-old-space-size=8192
353 ExecStart=/usr/bin/node dist/index.js
354 ExecReload=/bin/kill -HUP \$MAINPID
355 Restart=always
356 RestartSec=10
357 StandardOutput=journal
358 StandardError=journal
359
360 # Hardening
361 NoNewPrivileges=true
362 ProtectSystem=strict
363 ProtectHome=true
364 ReadWritePaths=$PAPERCLIP_INSTALL_DIR /var/lib/paperclip /tmp
365 PrivateTmp=true
366 PrivateDevices=true
367 ProtectKernelTunables=true
368 ProtectKernelModules=true
369 ProtectControlGroups=true
370 RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
371 RestrictRealtime=true
372 LockPersonality=true
373
374 [Install]
375 WantedBy=multi-user.target
376 EOF
377
378 mkdir -p /var/lib/paperclip
379 chown -R "$PAPERCLIP_USER:$PAPERCLIP_USER" /var/lib/paperclip
380
381 systemctl daemon-reload
382 systemctl enable --now paperclip-secrets-sync.timer
383 systemctl start paperclip-secrets-sync.service
384
385 # Wait for the first secrets sync.
386 echo "[install] Waiting up to 60 seconds for first SSM secrets sync..."
387 for i in $(seq 1 60); do
388 [[ -s /etc/paperclip/env ]] && break
389 sleep 1
390 done
391
392 if [[ ! -s /etc/paperclip/env ]]; then
393 echo "[install] WARNING: /etc/paperclip/env is empty after 60 seconds."
394 echo "[install] This is expected on first boot — push secrets via:"
395 echo " sudo -u $PAPERCLIP_USER /opt/paperclip/scripts/push-secrets.sh"
396 echo "[install] Skipping paperclip.service start until secrets are present."
397 else
398 systemctl enable --now paperclip.service
399 echo "[install] paperclip.service started"
400 fi
401
402 echo "[install] DONE. Next:"
403 echo " 1. sudo -u $PAPERCLIP_USER /opt/paperclip/scripts/push-secrets.sh # interactive"
404 echo " 2. sudo -u $PAPERCLIP_USER /opt/paperclip/scripts/hello-world-test.sh"
405 echo " 3. sudo -u $PAPERCLIP_USER /opt/paperclip/scripts/wire-knowtation-mcp.sh"
406 echo " 4. sudo -u $PAPERCLIP_USER /opt/paperclip/scripts/load-skills-and-agents.sh"
407 echo "Logs: journalctl -u paperclip.service -f"
408 echo "Dashboard: http://paperclip-prod (Tailscale-only by default)"