import { generateKeyPairSync } from "crypto"; import { db, bootstrapJobs } from "@workspace/db"; import { eq } from "drizzle-orm"; import { makeLogger } from "./logger.js"; const logger = makeLogger("provisioner"); const DO_API_BASE = "https://api.digitalocean.com/v2"; const TS_API_BASE = "https://api.tailscale.com/api/v2"; // ── SSH keypair via node:crypto ─────────────────────────────────────────────── function uint32BE(n: number): Buffer { const b = Buffer.allocUnsafe(4); b.writeUInt32BE(n, 0); return b; } function sshEncodeString(s: string): Buffer { const data = Buffer.from(s, "utf8"); return Buffer.concat([uint32BE(data.length), data]); } /** SSH mpint: prepend 0x00 if high bit set (indicates positive). */ function sshEncodeMpint(data: Buffer): Buffer { if (data[0] & 0x80) data = Buffer.concat([Buffer.from([0x00]), data]); return Buffer.concat([uint32BE(data.length), data]); } function derReadLength(buf: Buffer, offset: number): { len: number; offset: number } { if (!(buf[offset] & 0x80)) return { len: buf[offset], offset: offset + 1 }; const nb = buf[offset] & 0x7f; let len = 0; for (let i = 0; i < nb; i++) len = (len << 8) | buf[offset + 1 + i]; return { len, offset: offset + 1 + nb }; } function derReadInteger(buf: Buffer, offset: number): { value: Buffer; offset: number } { if (buf[offset] !== 0x02) throw new Error(`Expected DER INTEGER tag at ${offset}`); offset += 1; const { len, offset: dataStart } = derReadLength(buf, offset); return { value: buf.slice(dataStart, dataStart + len), offset: dataStart + len }; } /** Convert PKCS#1 DER RSA public key → OpenSSH wire format string. */ function pkcs1DerToSshPublicKey(der: Buffer): string { // Structure: SEQUENCE { INTEGER(n), INTEGER(e) } let offset = 0; if (der[offset] !== 0x30) throw new Error("Expected DER SEQUENCE"); offset += 1; const { offset: seqBody } = derReadLength(der, offset); offset = seqBody; const { value: n, offset: o2 } = derReadInteger(der, offset); const { value: e } = derReadInteger(der, o2); const payload = Buffer.concat([ sshEncodeString("ssh-rsa"), sshEncodeMpint(e), sshEncodeMpint(n), ]); return `ssh-rsa ${payload.toString("base64")} timmy-bootstrap-node`; } interface SshKeypair { privateKey: string; publicKey: string; } function generateSshKeypair(): SshKeypair { const { publicKey: pubDer, privateKey: privPem } = generateKeyPairSync("rsa", { modulusLength: 4096, publicKeyEncoding: { type: "pkcs1", format: "der" }, privateKeyEncoding: { type: "pkcs1", format: "pem" }, }); return { privateKey: privPem as string, publicKey: pkcs1DerToSshPublicKey(pubDer as unknown as Buffer), }; } // ── Cloud-init script ───────────────────────────────────────────────────────── function buildCloudInitScript(tailscaleAuthKey: string): string { const tsBlock = tailscaleAuthKey ? `tailscale up --authkey="${tailscaleAuthKey}" --ssh --accept-routes` : "# No Tailscale auth key — Tailscale not joined"; return `#!/bin/bash set -euo pipefail exec >> /var/log/timmy-bootstrap.log 2>&1 echo "[timmy] Bootstrap started at $(date -u)" # ── 1. Packages ─────────────────────────────────────────────── export DEBIAN_FRONTEND=noninteractive apt-get update -qq apt-get install -y -qq curl wget ufw jq openssl # ── 2. Docker ───────────────────────────────────────────────── if ! command -v docker &>/dev/null; then curl -fsSL https://get.docker.com | sh systemctl enable docker systemctl start docker fi # ── 3. Tailscale ────────────────────────────────────────────── if ! command -v tailscale &>/dev/null; then curl -fsSL https://tailscale.com/install.sh | sh fi ${tsBlock} # ── 4. Firewall ─────────────────────────────────────────────── ufw --force reset ufw allow in on tailscale0 ufw allow 8333/tcp ufw allow 9735/tcp ufw allow 22/tcp ufw default deny incoming ufw default allow outgoing ufw --force enable # ── 5. Block volume ─────────────────────────────────────────── mkdir -p /data VOLUME_DEV=$(lsblk -rno NAME,SIZE,MOUNTPOINT | awk '$3=="" && $2~/G/ {print $1}' | grep -vE "^(s|v)da$" | head -1 || true) if [[ -n "$VOLUME_DEV" ]]; then VOLUME_PATH="/dev/$VOLUME_DEV" if ! blkid "$VOLUME_PATH" &>/dev/null; then mkfs.ext4 -F "$VOLUME_PATH" fi mount "$VOLUME_PATH" /data BLKID=$(blkid -s UUID -o value "$VOLUME_PATH") grep -q "$BLKID" /etc/fstab || echo "UUID=$BLKID /data ext4 defaults,nofail 0 2" >> /etc/fstab echo "[timmy] Block volume mounted at /data ($VOLUME_PATH)" else echo "[timmy] No block volume — using /data on root disk" fi # ── 6. Directories ──────────────────────────────────────────── mkdir -p /data/bitcoin /data/lnd /data/lnbits /opt/timmy-node/configs # ── 7. Credentials ──────────────────────────────────────────── RPC_PASS=$(openssl rand -hex 24) LND_WALLET_PASS=$(openssl rand -hex 16) echo "[timmy] Credentials generated" # ── 8. Bitcoin config ───────────────────────────────────────── cat > /data/bitcoin/bitcoin.conf < /opt/timmy-node/configs/lnd.conf < /opt/timmy-node/docker-compose.yml </dev/null 2>&1; then echo "[timmy] Bitcoin RPC ready (\${i}x5s)" break fi sleep 5 done # ── 12. Start LND ───────────────────────────────────────────── docker compose up -d lnd echo "[timmy] LND started" echo "[timmy] Waiting for LND REST API..." for i in $(seq 1 72); do if curl -sk https://localhost:8080/v1/state >/dev/null 2>&1; then echo "[timmy] LND REST ready (\${i}x5s)" break fi sleep 5 done # ── 13. Init LND wallet (non-interactive via REST) ──────────── echo "[timmy] Generating LND wallet seed..." SEED_RESP=$(curl -sk https://localhost:8080/v1/genseed) SEED_JSON=$(echo "$SEED_RESP" | jq '.cipher_seed_mnemonic') SEED_WORDS=$(echo "$SEED_JSON" | jq -r 'join(" ")') PASS_B64=$(printf '%s' "$LND_WALLET_PASS" | base64 -w0) echo "[timmy] Initializing LND wallet..." INIT_RESP=$(curl -sk -X POST https://localhost:8080/v1/initwallet \ -H "Content-Type: application/json" \ -d "{\"wallet_password\": \"$PASS_B64\", \"cipher_seed_mnemonic\": $SEED_JSON}") echo "[timmy] Wallet init: $(echo "$INIT_RESP" | jq -r 'if .admin_macaroon then "ok" else tostring end')" echo "[timmy] Waiting for admin macaroon..." for i in $(seq 1 60); do if [[ -f /data/lnd/data/chain/bitcoin/mainnet/admin.macaroon ]]; then echo "[timmy] Admin macaroon ready (\${i}x5s)" break fi sleep 5 done # ── 14. Start LNbits ────────────────────────────────────────── docker compose up -d lnbits echo "[timmy] LNbits started" echo "[timmy] Waiting for LNbits..." for i in $(seq 1 36); do if curl -s http://localhost:3000/health >/dev/null 2>&1; then echo "[timmy] LNbits ready (\${i}x5s)" break fi sleep 5 done # ── 15. Install ops helper ──────────────────────────────────── cat > /opt/timmy-node/ops.sh <<'OPSSH' #!/bin/bash CMD=\${1:-help} case "\$CMD" in sync) echo "=== Bitcoin Sync Status ===" docker exec bitcoin bitcoin-cli -datadir=/home/bitcoin/.bitcoin getblockchaininfo 2>&1 \ | jq '{chain, blocks, headers, progress: (.verificationprogress*100|round|tostring+"%"), pruned}' ;; lnd) docker exec lnd lncli --network=mainnet getinfo 2>&1 ;; lnbits) curl -s http://localhost:3000/health && echo "" ;; logs) docker logs --tail 80 "\${2:-bitcoin}" ;; help|*) echo "Usage: bash /opt/timmy-node/ops.sh " echo " sync — Bitcoin sync progress (1-2 weeks to 100%)" echo " lnd — LND node info" echo " lnbits — LNbits health check" echo " logs [svc] — Recent logs for bitcoin | lnd | lnbits" ;; esac OPSSH chmod +x /opt/timmy-node/ops.sh echo "[timmy] ops.sh installed at /opt/timmy-node/ops.sh" # ── 16. Save credentials ────────────────────────────────────── NODE_IP=$(curl -4s https://ifconfig.me 2>/dev/null || echo "unknown") cat > /root/node-credentials.txt <(endpoint: string, token: string, body: unknown): Promise { const res = await fetch(`${DO_API_BASE}${endpoint}`, { method: "POST", headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, body: JSON.stringify(body), }); if (!res.ok) { const text = await res.text(); throw new Error(`DO API POST ${endpoint} failed (${res.status}): ${text}`); } return res.json() as Promise; } async function doGet(endpoint: string, token: string): Promise { const res = await fetch(`${DO_API_BASE}${endpoint}`, { headers: { Authorization: `Bearer ${token}` }, }); if (!res.ok) { const text = await res.text(); throw new Error(`DO API GET ${endpoint} failed (${res.status}): ${text}`); } return res.json() as Promise; } async function pollDropletIp(dropletId: number, token: string, maxMs = 120_000): Promise { const deadline = Date.now() + maxMs; while (Date.now() < deadline) { await new Promise((r) => setTimeout(r, 5000)); const data = await doGet<{ droplet: { networks: { v4: Array<{ type: string; ip_address: string }> } }; }>(`/droplets/${dropletId}`, token); const pub = data.droplet?.networks?.v4?.find((n) => n.type === "public"); if (pub?.ip_address) return pub.ip_address; } return null; } async function createVolume( name: string, sizeGb: number, region: string, token: string, ): Promise { const data = await doPost<{ volume: { id: string } }>("/volumes", token, { name, size_gigabytes: sizeGb, region, filesystem_type: "ext4", description: "Timmy node data volume", tags: ["timmy-node"], }); return data.volume.id; } // ── Tailscale helper ────────────────────────────────────────────────────────── async function getTailscaleAuthKey(apiKey: string, tailnet: string): Promise { const res = await fetch(`${TS_API_BASE}/tailnet/${tailnet}/keys`, { method: "POST", headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }, body: JSON.stringify({ capabilities: { devices: { create: { reusable: false, ephemeral: false, preauthorized: true, tags: ["tag:timmy-node"] }, }, }, expirySeconds: 86400, description: "timmy-bootstrap", }), }); if (!res.ok) { const text = await res.text(); throw new Error(`Tailscale API failed (${res.status}): ${text}`); } const data = (await res.json()) as { key: string }; return data.key; } // ── ProvisionerService ──────────────────────────────────────────────────────── export class ProvisionerService { readonly stubMode: boolean; private readonly doToken: string; private readonly doRegion: string; private readonly doSize: string; private readonly doVolumeGb: number; private readonly tsApiKey: string; private readonly tsTailnet: string; constructor() { this.doToken = process.env.DO_API_TOKEN ?? ""; this.doRegion = process.env.DO_REGION ?? "nyc3"; this.doSize = process.env.DO_SIZE ?? "s-4vcpu-8gb"; this.doVolumeGb = parseInt(process.env.DO_VOLUME_SIZE_GB ?? "0", 10) || 0; this.tsApiKey = process.env.TAILSCALE_API_KEY ?? ""; this.tsTailnet = process.env.TAILSCALE_TAILNET ?? ""; this.stubMode = !this.doToken; if (this.stubMode) { logger.warn("no DO_API_TOKEN — running in STUB mode", { stub: true }); } } /** * Fire-and-forget: call without awaiting. * Updates bootstrap_jobs.state to ready/failed when complete. */ async provision(bootstrapJobId: string): Promise { try { if (this.stubMode) { await this.stubProvision(bootstrapJobId); } else { await this.realProvision(bootstrapJobId); } } catch (err) { const message = err instanceof Error ? err.message : "Provisioning failed"; logger.error("provisioning failed", { bootstrapJobId, error: message }); await db .update(bootstrapJobs) .set({ state: "failed", errorMessage: message, updatedAt: new Date() }) .where(eq(bootstrapJobs.id, bootstrapJobId)); } } private async stubProvision(jobId: string): Promise { logger.info("stub provisioning started", { bootstrapJobId: jobId }); const { privateKey } = generateSshKeypair(); await new Promise((r) => setTimeout(r, 2000)); const fakeDropletId = String(Math.floor(Math.random() * 900_000_000 + 100_000_000)); await db .update(bootstrapJobs) .set({ state: "ready", dropletId: fakeDropletId, nodeIp: "198.51.100.42", tailscaleHostname: `timmy-node-${jobId.slice(0, 8)}.tail1234.ts.net`, lnbitsUrl: `http://timmy-node-${jobId.slice(0, 8)}.tail1234.ts.net:3000`, sshPrivateKey: privateKey, updatedAt: new Date(), }) .where(eq(bootstrapJobs.id, jobId)); logger.info("stub provisioning complete", { bootstrapJobId: jobId }); } private async realProvision(jobId: string): Promise { logger.info("real provisioning started", { bootstrapJobId: jobId }); // 1. SSH keypair (pure node:crypto) const { publicKey, privateKey } = generateSshKeypair(); // 2. Upload public key to DO const keyName = `timmy-bootstrap-${jobId.slice(0, 8)}`; const keyData = await doPost<{ ssh_key: { id: number } }>("/account/keys", this.doToken, { name: keyName, public_key: publicKey, }); const sshKeyId = keyData.ssh_key.id; // 3. Tailscale auth key (optional) let tailscaleAuthKey = ""; if (this.tsApiKey && this.tsTailnet) { try { tailscaleAuthKey = await getTailscaleAuthKey(this.tsApiKey, this.tsTailnet); } catch (err) { logger.warn("Tailscale key failed — continuing without Tailscale", { error: String(err) }); } } // 4. Create block volume if configured let volumeId: string | null = null; if (this.doVolumeGb > 0) { const volName = `timmy-data-${jobId.slice(0, 8)}`; volumeId = await createVolume(volName, this.doVolumeGb, this.doRegion, this.doToken); logger.info("block volume created", { volumeId, sizeGb: this.doVolumeGb }); } // 5. Create droplet const userData = buildCloudInitScript(tailscaleAuthKey); const dropletPayload: Record = { name: `timmy-node-${jobId.slice(0, 8)}`, region: this.doRegion, size: this.doSize, image: "ubuntu-22-04-x64", ssh_keys: [sshKeyId], user_data: userData, tags: ["timmy-node"], }; if (volumeId) dropletPayload.volumes = [volumeId]; const dropletData = await doPost<{ droplet: { id: number } }>( "/droplets", this.doToken, dropletPayload, ); const dropletId = dropletData.droplet.id; logger.info("droplet created", { bootstrapJobId: jobId, dropletId }); // 6. Poll for public IP (up to 2 min) const nodeIp = await pollDropletIp(dropletId, this.doToken, 120_000); logger.info("node ip assigned", { bootstrapJobId: jobId, nodeIp: nodeIp ?? "(not yet assigned)" }); // 7. Tailscale hostname const tailscaleHostname = tailscaleAuthKey && this.tsTailnet ? `timmy-node-${jobId.slice(0, 8)}.${this.tsTailnet}.ts.net` : null; // LNbits listens on port 3000 (HTTP). Tailscale encrypts the link at the // network layer, so http:// is correct — no TLS termination on the service. const lnbitsUrl = tailscaleHostname ? `http://${tailscaleHostname}:3000` : nodeIp ? `http://${nodeIp}:3000` : null; await db .update(bootstrapJobs) .set({ state: "ready", dropletId: String(dropletId), nodeIp, tailscaleHostname, lnbitsUrl, sshPrivateKey: privateKey, updatedAt: new Date(), }) .where(eq(bootstrapJobs.id, jobId)); logger.info("real provisioning complete", { bootstrapJobId: jobId }); } } export const provisionerService = new ProvisionerService();