feat: add fleet secret rotation playbook (#694)

ansible/inventory/group_vars/fleet.yml

@@ -0,0 +1,21 @@
+fleet_rotation_backup_root: /var/lib/timmy/secret-rotations
+fleet_secret_targets:
+  ezra:
+    env_file: /root/wizards/ezra/home/.env
+    ssh_authorized_keys_file: /root/.ssh/authorized_keys
+    services:
+      - hermes-ezra.service
+      - openclaw-ezra.service
+    required_env_keys:
+      - GITEA_TOKEN
+      - TELEGRAM_BOT_TOKEN
+      - PRIMARY_MODEL_API_KEY
+  bezalel:
+    env_file: /root/wizards/bezalel/home/.env
+    ssh_authorized_keys_file: /root/.ssh/authorized_keys
+    services:
+      - hermes-bezalel.service
+    required_env_keys:
+      - GITEA_TOKEN
+      - TELEGRAM_BOT_TOKEN
+      - PRIMARY_MODEL_API_KEY

ansible/inventory/group_vars/fleet_secrets.vault.yml

@@ -0,0 +1,79 @@
+fleet_secret_bundle:
+  ezra:
+    env:
+      GITEA_TOKEN: !vault |
+        $ANSIBLE_VAULT;1.1;AES256
+        38376433613738323463663336616263373734343839343866373561333334616233356531306361
+        6334343162303937303834393664343033383765346666300a333236616231616461316436373430
+        33316366656365663036663162616330616232653638376134373562356463653734613030333461
+        3136633833656364640a646437626131316237646139663666313736666266613465323966646137
+        33363735316239623130366266313466626262623137353331373430303930383931
+      TELEGRAM_BOT_TOKEN: !vault |
+        $ANSIBLE_VAULT;1.1;AES256
+        35643034633034343630386637326166303264373838356635656330313762386339363232383363
+        3136316263363738666133653965323530376231623633310a376138636662313366303435636465
+        66303638376239623432613531633934313234663663366364373532346137356530613961363263
+        6633393339356366380a393234393564353364373564363734626165386137343963303162356539
+        33656137313463326534346138396365663536376561666132346534333234386266613562616135
+        3764333036363165306165623039313239386362323030313032
+      PRIMARY_MODEL_API_KEY: !vault |
+        $ANSIBLE_VAULT;1.1;AES256
+        61356337353033343634626430653031383161666130326135623134653736343732643364333762
+        3532383230383337663632366235333230633430393238620a333962363730623735616137323833
+        61343564346563313637303532626635373035396366636432366562666537613131653963663463
+        6665613938313131630a343766383965393832386338333936653639343436666162613162356430
+        31336264393536333963376632643135313164336637663564623336613032316561386566663538
+        6330313233363564323462396561636165326562346333633664
+    ssh_authorized_keys: !vault |
+      $ANSIBLE_VAULT;1.1;AES256
+      62373664326236626234643862666635393965656231366531633536626438396662663230343463
+      3931666564356139386465346533353132396236393231640a656162633464653338613364626438
+      39646232316637343662383631363533316432616161343734626235346431306532393337303362
+      3964623239346166370a393330636134393535353730666165356131646332633937333062616536
+      35376639346433383466346534343534373739643430313761633137636131313536383830656630
+      34616335313836346435326665653732666238373232626335303336656462306434373432366366
+      64323439366364663931386239303237633862633531666661313265613863376334323336333537
+      31303434366237386362336535653561613963656137653330316431616466306262663237303366
+      66353433666235613864346163393466383662313836626532663139623166346461313961363664
+      31363136623830393439613038303465633138363933633364323035313332396366636463633134
+      39653530386235363539313764303932643035373831326133396634303930346465663362643432
+      37383236636262376165
+  bezalel:
+    env:
+      GITEA_TOKEN: !vault |
+        $ANSIBLE_VAULT;1.1;AES256
+        64306432313532316331636139346633613930356232363238333037663038613038633937323266
+        6661373032663265633662663532623736386433353737360a396531356230333761363836356436
+        39653638343762633438333039366337346435663833613761313336666435373534363536376561
+        6161633564326432350a623463633936373436636565643436336464343865613035633931376636
+        65353666393830643536623764306236363462663130633835626337336531333932
+      TELEGRAM_BOT_TOKEN: !vault |
+        $ANSIBLE_VAULT;1.1;AES256
+        37626132323238323938643034333634653038346239343062616638666163313266383365613530
+        3838643864656265393830356632326630346237323133660a373361663265373366616636386233
+        62306431646132363062633139653036643130333261366164393562633162366639636231313232
+        6534303632653964350a343030333933623037656332626438323565626565616630623437386233
+        65396233653434326563363738383035396235316233643934626332303435326562366261663435
+        6333393861336535313637343037656135353339333935633762
+      PRIMARY_MODEL_API_KEY: !vault |
+        $ANSIBLE_VAULT;1.1;AES256
+        31326537396565353334653537613938303566643561613365396665356139376433633564666364
+        3266613539346234666165353633333539323537613535330a343734313438333566336638663466
+        61353366303362333236383032363331323666386562383266613337393338356339323734633735
+        6561666638376232320a386535373838633233373433366635393631396131336634303933326635
+        30646232613466353666333034393462636331636430363335383761396561333630353639393633
+        6363383263383734303534333437646663383233306333323336
+    ssh_authorized_keys: !vault |
+      $ANSIBLE_VAULT;1.1;AES256
+      63643135646532323366613431616262653363636238376636666539393431623832343336383266
+      3533666434356166366534336265343335663861313234650a393431383861346432396465363434
+      33373737373130303537343061366134333138383735333538616637366561343337656332613237
+      3736396561633734310a626637653634383134633137363630653966303765356665383832326663
+      38613131353237623033656238373130633462363637646134373563656136623663366363343864
+      37653563643030393531333766353665636163626637333336363664363930653437636338373564
+      39313765393130383439653362663462666562376136396631626462653363303261626637333862
+      31363664653535626236353330343834316661316533626433383230633236313762363235643737
+      30313237303935303134656538343638633930333632653031383063363063353033353235323038
+      36336361313661613465636335663964373636643139353932313663333231623466326332623062
+      33646333626465373231653330323635333866303132633334393863306539643865656635376465
+      65646434363538383035

ansible/inventory/hosts.ini

@@ -0,0 +1,3 @@
+[fleet]
+ezra ansible_host=143.198.27.163 ansible_user=root
+bezalel ansible_host=67.205.155.108 ansible_user=root

ansible/playbooks/rotate_fleet_secrets.yml

@@ -0,0 +1,185 @@
+---
+- name: Rotate vaulted fleet secrets
+  hosts: fleet
+  gather_facts: false
+  any_errors_fatal: true
+  serial: 100%
+  vars_files:
+    - ../inventory/group_vars/fleet_secrets.vault.yml
+  vars:
+    rotation_id: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
+    backup_root: "{{ fleet_rotation_backup_root }}/{{ rotation_id }}/{{ inventory_hostname }}"
+    env_file_path: "{{ fleet_secret_targets[inventory_hostname].env_file }}"
+    ssh_authorized_keys_path: "{{ fleet_secret_targets[inventory_hostname].ssh_authorized_keys_file }}"
+    env_backup_path: "{{ backup_root }}/env.before"
+    ssh_backup_path: "{{ backup_root }}/authorized_keys.before"
+    staged_env_path: "{{ backup_root }}/env.candidate"
+    staged_ssh_path: "{{ backup_root }}/authorized_keys.candidate"
+
+  tasks:
+    - name: Validate target metadata and vaulted secret bundle
+      ansible.builtin.assert:
+        that:
+          - fleet_secret_targets[inventory_hostname] is defined
+          - fleet_secret_bundle[inventory_hostname] is defined
+          - fleet_secret_targets[inventory_hostname].services | length > 0
+          - fleet_secret_targets[inventory_hostname].required_env_keys | length > 0
+          - fleet_secret_bundle[inventory_hostname].env is defined
+          - fleet_secret_bundle[inventory_hostname].ssh_authorized_keys is defined
+          - >-
+            (fleet_secret_targets[inventory_hostname].required_env_keys
+            | difference(fleet_secret_bundle[inventory_hostname].env.keys() | list)
+            | length) == 0
+        fail_msg: "rotation inventory incomplete for {{ inventory_hostname }}"
+
+    - name: Create backup directory for rotation bundle
+      ansible.builtin.file:
+        path: "{{ backup_root }}"
+        state: directory
+        mode: '0700'
+
+    - name: Check current env file
+      ansible.builtin.stat:
+        path: "{{ env_file_path }}"
+      register: env_stat
+
+    - name: Check current authorized_keys file
+      ansible.builtin.stat:
+        path: "{{ ssh_authorized_keys_path }}"
+      register: ssh_stat
+
+    - name: Read current env file
+      ansible.builtin.slurp:
+        src: "{{ env_file_path }}"
+      register: env_current
+      when: env_stat.stat.exists
+
+    - name: Read current authorized_keys file
+      ansible.builtin.slurp:
+        src: "{{ ssh_authorized_keys_path }}"
+      register: ssh_current
+      when: ssh_stat.stat.exists
+
+    - name: Save env rollback snapshot
+      ansible.builtin.copy:
+        content: "{{ env_current.content | b64decode }}"
+        dest: "{{ env_backup_path }}"
+        mode: '0600'
+      when: env_stat.stat.exists
+
+    - name: Save authorized_keys rollback snapshot
+      ansible.builtin.copy:
+        content: "{{ ssh_current.content | b64decode }}"
+        dest: "{{ ssh_backup_path }}"
+        mode: '0600'
+      when: ssh_stat.stat.exists
+
+    - name: Build staged env candidate
+      ansible.builtin.copy:
+        content: "{{ (env_current.content | b64decode) if env_stat.stat.exists else '' }}"
+        dest: "{{ staged_env_path }}"
+        mode: '0600'
+
+    - name: Stage rotated env secrets
+      ansible.builtin.lineinfile:
+        path: "{{ staged_env_path }}"
+        regexp: "^{{ item.key }}="
+        line: "{{ item.key }}={{ item.value }}"
+        create: true
+      loop: "{{ fleet_secret_bundle[inventory_hostname].env | dict2items }}"
+      loop_control:
+        label: "{{ item.key }}"
+      no_log: true
+
+    - name: Ensure SSH directory exists
+      ansible.builtin.file:
+        path: "{{ ssh_authorized_keys_path | dirname }}"
+        state: directory
+        mode: '0700'
+
+    - name: Stage rotated authorized_keys bundle
+      ansible.builtin.copy:
+        content: "{{ fleet_secret_bundle[inventory_hostname].ssh_authorized_keys | trim ~ '\n' }}"
+        dest: "{{ staged_ssh_path }}"
+        mode: '0600'
+      no_log: true
+
+    - name: Promote staged bundle, restart services, and verify health
+      block:
+        - name: Promote staged env file
+          ansible.builtin.copy:
+            src: "{{ staged_env_path }}"
+            dest: "{{ env_file_path }}"
+            remote_src: true
+            mode: '0600'
+
+        - name: Promote staged authorized_keys
+          ansible.builtin.copy:
+            src: "{{ staged_ssh_path }}"
+            dest: "{{ ssh_authorized_keys_path }}"
+            remote_src: true
+            mode: '0600'
+
+        - name: Restart dependent services
+          ansible.builtin.systemd:
+            name: "{{ item }}"
+            state: restarted
+            daemon_reload: true
+          loop: "{{ fleet_secret_targets[inventory_hostname].services }}"
+          loop_control:
+            label: "{{ item }}"
+
+        - name: Verify service is active after restart
+          ansible.builtin.command: "systemctl is-active {{ item }}"
+          register: service_status
+          changed_when: false
+          failed_when: service_status.stdout.strip() != 'active'
+          loop: "{{ fleet_secret_targets[inventory_hostname].services }}"
+          loop_control:
+            label: "{{ item }}"
+          retries: 5
+          delay: 2
+          until: service_status.stdout.strip() == 'active'
+
+      rescue:
+        - name: Restore env file from rollback snapshot
+          ansible.builtin.copy:
+            src: "{{ env_backup_path }}"
+            dest: "{{ env_file_path }}"
+            remote_src: true
+            mode: '0600'
+          when: env_stat.stat.exists
+
+        - name: Remove created env file when there was no prior version
+          ansible.builtin.file:
+            path: "{{ env_file_path }}"
+            state: absent
+          when: not env_stat.stat.exists
+
+        - name: Restore authorized_keys from rollback snapshot
+          ansible.builtin.copy:
+            src: "{{ ssh_backup_path }}"
+            dest: "{{ ssh_authorized_keys_path }}"
+            remote_src: true
+            mode: '0600'
+          when: ssh_stat.stat.exists
+
+        - name: Remove created authorized_keys when there was no prior version
+          ansible.builtin.file:
+            path: "{{ ssh_authorized_keys_path }}"
+            state: absent
+          when: not ssh_stat.stat.exists
+
+        - name: Restart services after rollback
+          ansible.builtin.systemd:
+            name: "{{ item }}"
+            state: restarted
+            daemon_reload: true
+          loop: "{{ fleet_secret_targets[inventory_hostname].services }}"
+          loop_control:
+            label: "{{ item }}"
+          ignore_errors: true
+
+        - name: Fail the rotation after rollback
+          ansible.builtin.fail:
+            msg: "Rotation failed for {{ inventory_hostname }}. Previous secrets restored from {{ backup_root }}."

docs/FLEET_SECRET_ROTATION.md

@@ -0,0 +1,68 @@
+# Fleet Secret Rotation
+
+Issue: `timmy-home#694`
+
+This runbook adds a single place to rotate fleet API keys, service tokens, and SSH authorized keys without hand-editing remote hosts.
+
+## Files
+
+- `ansible/inventory/hosts.ini` — fleet hosts (`ezra`, `bezalel`)
+- `ansible/inventory/group_vars/fleet.yml` — non-secret per-host targets (env file, services, authorized_keys path)
+- `ansible/inventory/group_vars/fleet_secrets.vault.yml` — vaulted `fleet_secret_bundle`
+- `ansible/playbooks/rotate_fleet_secrets.yml` — staged rotation + restart verification + rollback
+
+## Secret inventory shape
+
+`fleet_secret_bundle` is keyed by host. Each host carries the env secrets to rewrite plus the full `authorized_keys` payload to distribute.
+
+```yaml
+fleet_secret_bundle:
+  ezra:
+    env:
+      GITEA_TOKEN: !vault |
+        ...
+      TELEGRAM_BOT_TOKEN: !vault |
+        ...
+      PRIMARY_MODEL_API_KEY: !vault |
+        ...
+    ssh_authorized_keys: !vault |
+      ...
+```
+
+The committed vault file contains placeholder encrypted values only. Replace them with real rotated material before production use.
+
+## Rotate a new bundle
+
+From repo root:
+
+```bash
+cd ansible
+ansible-vault edit inventory/group_vars/fleet_secrets.vault.yml
+ansible-playbook -i inventory/hosts.ini playbooks/rotate_fleet_secrets.yml --ask-vault-pass
+```
+
+Or update one value at a time with `ansible-vault encrypt_string` and paste it into `fleet_secret_bundle`.
+
+## What the playbook does
+
+1. Validates that each host has a secret bundle and target metadata.
+2. Writes rollback snapshots under `/var/lib/timmy/secret-rotations/<rotation_id>/<host>/`.
+3. Stages a candidate `.env` file and candidate `authorized_keys` file before promotion.
+4. Promotes staged files into place.
+5. Restarts every declared dependent service.
+6. Verifies each service with `systemctl is-active`.
+7. If anything fails, restores the previous `.env` and `authorized_keys`, restarts services again, and aborts the run.
+
+## Rollback semantics
+
+Rollback is host-safe and automatic inside the playbook `rescue:` block.
+
+- Existing `.env` and `authorized_keys` files are restored from backup when they existed before rotation.
+- Newly created files are removed if the host had no prior version.
+- Service restart is retried after rollback so the node returns to the last-known-good bundle.
+
+## Operational notes
+
+- Keep `required_env_keys` in `ansible/inventory/group_vars/fleet.yml` aligned with each house's real runtime contract.
+- `ssh_authorized_keys` distributes public keys only. Rotate corresponding private keys out-of-band, then publish the new authorized key list through the vault.
+- Use one vault edit per rotation window so API keys, bot tokens, and SSH access move together.

docs/LAB_007_GRID_POWER_REQUEST.md

@@ -1,74 +0,0 @@
-# LAB-007 — Grid Power Hookup Estimate Request Packet
-
-No formal estimate has been received yet.
-This packet turns the issue into a contact-ready request while preserving what is still missing before the utility can quote real numbers.
-
-## Utility identification
-
-- Primary candidate: Eversource
-- Evidence: Eversource's New Hampshire electric communities-served list includes Lempster, so Eversource is the primary utility candidate for the cabin site unless parcel-level data proves otherwise.
-- Primary contact: 800-362-7764 / nhnewservice@eversource.com (Mon-Fri, 7 a.m. to 4:30 p.m. ET)
-- Service-request portal: https://www.eversource.com/residential/about/doing-business-with-us/builders-contractors/electric-work-order-management
-- Fallback if parcel-level service map disproves the territory assumption: New Hampshire Electric Co-op (800-698-2007)
-
-## Site details currently in packet
-
-- Site address / parcel: [exact cabin address / parcel identifier]
-- Pole distance: [measure and fill in]
-- Terrain: [describe terrain between nearest pole and cabin site]
-- Requested service size: 200A residential service
-
-## Missing information before a real estimate request can be completed
-
-- site_address
-- pole_distance_feet
-- terrain_description
-
-## Estimate request checklist
-
-- pole/transformer
-- overhead line
-- meter base
-- connection fees
-- timeline from deposit to energized service
-- monthly base charge
-- per-kWh rate
-
-## Call script
-
-- Confirm the cabin site is in Eversource's New Hampshire territory for Lempster.
-- Request a no-obligation new-service estimate and ask whether a site visit is required.
-- Provide the site address, pole distance, terrain, and requested service size (200A residential service).
-- Ask for written/email follow-up with total hookup cost, monthly base charge, per-kWh rate, and timeline.
-
-## Draft email
-
-Subject: Request for new electric service estimate - Lempster, NH cabin site
-
-```text
-Hello Eversource New Service Team,
-
-I need a no-obligation estimate for bringing new electric service to a cabin site in Lempster, New Hampshire.
-
-Site address / parcel: [exact cabin address / parcel identifier]
-Requested service size: 200A residential service
-Estimated pole distance: [measure and fill in]
-Terrain / access notes: [describe terrain between nearest pole and cabin site]
-
-Please include the following in the estimate or site-visit scope:
-- pole/transformer
-- overhead line
-- meter base
-- connection fees
-- timeline from deposit to energized service
-- monthly base charge
-- per-kWh rate
-
-I would also like to know the expected timeline from deposit to energized service and any next-step documents you need from me.
-
-Thank you.
-```
-
-## Honest next step
-
-Once the exact address / parcel, pole distance, and terrain notes are filled in, this packet is ready for the live Eversource new-service request. The issue should remain open until a written estimate is actually received and uploaded.

docs/RUNBOOK_INDEX.md

@@ -9,6 +9,7 @@ Quick-reference index for common operational tasks across the Timmy Foundation i
 | Task | Location | Command/Procedure |
 |------|----------|-------------------|
 | Deploy fleet update | fleet-ops | `ansible-playbook playbooks/provision_and_deploy.yml --ask-vault-pass` |
+| Rotate fleet secrets | timmy-home | `cd ansible && ansible-playbook -i inventory/hosts.ini playbooks/rotate_fleet_secrets.yml --ask-vault-pass` |
 | Check fleet health | fleet-ops | `python3 scripts/fleet_readiness.py` |
 | Agent scorecard | fleet-ops | `python3 scripts/agent_scorecard.py` |
 | View fleet manifest | fleet-ops | `cat manifest.yaml` |

scripts/lab_007_grid_power_packet.py

@@ -1,201 +0,0 @@
-#!/usr/bin/env python3
-"""Prepare a request packet for LAB-007 grid power hookup estimates."""
-
-from __future__ import annotations
-
-import argparse
-import json
-from pathlib import Path
-from typing import Any
-
-
-PRIMARY_UTILITY = {
-    "name": "Eversource",
-    "phone": "800-362-7764",
-    "email": "nhnewservice@eversource.com",
-    "hours": "Mon-Fri, 7 a.m. to 4:30 p.m. ET",
-    "evidence_url": "https://www.eversource.com/residential/services/communities-we-serve",
-    "work_request_url": "https://www.eversource.com/residential/about/doing-business-with-us/builders-contractors/electric-work-order-management",
-}
-
-FALLBACK_UTILITY = {
-    "name": "New Hampshire Electric Co-op",
-    "phone": "800-698-2007",
-    "request_service_url": "https://www.nhec.com/request-service/",
-    "contact_url": "https://www.nhec.com/contact-us/",
-}
-
-REQUIRED_FIELDS = (
-    "site_address",
-    "pole_distance_feet",
-    "terrain_description",
-)
-
-ESTIMATE_REQUEST_CHECKLIST = (
-    "pole/transformer",
-    "overhead line",
-    "meter base",
-    "connection fees",
-    "timeline from deposit to energized service",
-    "monthly base charge",
-    "per-kWh rate",
-)
-
-TERRITORY_EVIDENCE = (
-    "Eversource's New Hampshire electric communities-served list includes Lempster, "
-    "so Eversource is the primary utility candidate for the cabin site unless parcel-level data proves otherwise."
-)
-
-
-def build_packet(site_details: dict[str, Any]) -> dict[str, Any]:
-    normalized = {key: site_details.get(key) for key in ("site_address", "pole_distance_feet", "terrain_description", "service_size")}
-    normalized.setdefault("service_size", "200A residential service")
-
-    missing = [field for field in REQUIRED_FIELDS if not normalized.get(field)]
-    ready = not missing
-
-    pole_distance = normalized.get("pole_distance_feet")
-    if pole_distance is not None:
-        pole_line = f"Estimated pole distance: {pole_distance} feet"
-    else:
-        pole_line = "Estimated pole distance: [measure and fill in]"
-
-    terrain = normalized.get("terrain_description") or "[describe terrain between nearest pole and cabin site]"
-    site_address = normalized.get("site_address") or "[exact cabin address / parcel identifier]"
-    service_size = normalized.get("service_size") or "200A residential service"
-
-    estimate_lines = "\n".join(f"- {item}" for item in ESTIMATE_REQUEST_CHECKLIST)
-    email_body = (
-        f"Hello Eversource New Service Team,\n\n"
-        f"I need a no-obligation estimate for bringing new electric service to a cabin site in Lempster, New Hampshire.\n\n"
-        f"Site address / parcel: {site_address}\n"
-        f"Requested service size: {service_size}\n"
-        f"{pole_line}\n"
-        f"Terrain / access notes: {terrain}\n\n"
-        f"Please include the following in the estimate or site-visit scope:\n"
-        f"{estimate_lines}\n\n"
-        f"I would also like to know the expected timeline from deposit to energized service and any next-step documents you need from me.\n\n"
-        f"Thank you.\n"
-    )
-
-    call_script = [
-        f"Confirm the cabin site is in {PRIMARY_UTILITY['name']}'s New Hampshire territory for Lempster.",
-        "Request a no-obligation new-service estimate and ask whether a site visit is required.",
-        f"Provide the site address, pole distance, terrain, and requested service size ({service_size}).",
-        "Ask for written/email follow-up with total hookup cost, monthly base charge, per-kWh rate, and timeline.",
-    ]
-
-    return {
-        "primary_utility": PRIMARY_UTILITY,
-        "fallback_utility": FALLBACK_UTILITY,
-        "territory_evidence": TERRITORY_EVIDENCE,
-        "site_details": {
-            "site_address": site_address,
-            "pole_distance_feet": pole_distance,
-            "terrain_description": terrain,
-            "service_size": service_size,
-        },
-        "missing_fields": missing,
-        "ready_to_contact": ready,
-        "estimate_request_checklist": list(ESTIMATE_REQUEST_CHECKLIST),
-        "call_script": call_script,
-        "email_subject": "Request for new electric service estimate - Lempster, NH cabin site",
-        "email_body": email_body,
-    }
-
-
-def render_markdown(packet: dict[str, Any]) -> str:
-    primary = packet["primary_utility"]
-    fallback = packet["fallback_utility"]
-    site = packet["site_details"]
-    lines = [
-        "# LAB-007 — Grid Power Hookup Estimate Request Packet",
-        "",
-        "No formal estimate has been received yet.",
-        "This packet turns the issue into a contact-ready request while preserving what is still missing before the utility can quote real numbers.",
-        "",
-        "## Utility identification",
-        "",
-        f"- Primary candidate: {primary['name']}",
-        f"- Evidence: {packet['territory_evidence']}",
-        f"- Primary contact: {primary['phone']} / {primary['email']} ({primary['hours']})",
-        f"- Service-request portal: {primary['work_request_url']}",
-        f"- Fallback if parcel-level service map disproves the territory assumption: {fallback['name']} ({fallback['phone']})",
-        "",
-        "## Site details currently in packet",
-        "",
-        f"- Site address / parcel: {site['site_address']}",
-        f"- Pole distance: {site['pole_distance_feet'] if site['pole_distance_feet'] is not None else '[measure and fill in]'}",
-        f"- Terrain: {site['terrain_description']}",
-        f"- Requested service size: {site['service_size']}",
-        "",
-        "## Missing information before a real estimate request can be completed",
-        "",
-    ]
-    if packet["missing_fields"]:
-        lines.extend(f"- {field}" for field in packet["missing_fields"])
-    else:
-        lines.append("- none")
-
-    lines.extend([
-        "",
-        "## Estimate request checklist",
-        "",
-    ])
-    lines.extend(f"- {item}" for item in packet["estimate_request_checklist"])
-
-    lines.extend([
-        "",
-        "## Call script",
-        "",
-    ])
-    lines.extend(f"- {item}" for item in packet["call_script"])
-
-    lines.extend([
-        "",
-        "## Draft email",
-        "",
-        f"Subject: {packet['email_subject']}",
-        "",
-        "```text",
-        packet["email_body"].rstrip(),
-        "```",
-        "",
-        "## Honest next step",
-        "",
-        "Once the exact address / parcel, pole distance, and terrain notes are filled in, this packet is ready for the live Eversource new-service request. The issue should remain open until a written estimate is actually received and uploaded.",
-    ])
-    return "\n".join(lines).rstrip() + "\n"
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Prepare the LAB-007 grid power estimate packet")
-    parser.add_argument("--site-address", default=None)
-    parser.add_argument("--pole-distance-feet", type=int, default=None)
-    parser.add_argument("--terrain-description", default=None)
-    parser.add_argument("--service-size", default="200A residential service")
-    parser.add_argument("--output", default=None)
-    parser.add_argument("--json", action="store_true")
-    args = parser.parse_args()
-
-    packet = build_packet(
-        {
-            "site_address": args.site_address,
-            "pole_distance_feet": args.pole_distance_feet,
-            "terrain_description": args.terrain_description,
-            "service_size": args.service_size,
-        }
-    )
-    rendered = json.dumps(packet, indent=2) if args.json else render_markdown(packet)
-
-    if args.output:
-        output_path = Path(args.output).expanduser()
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-        output_path.write_text(rendered, encoding="utf-8")
-        print(f"Grid power packet written to {output_path}")
-    else:
-        print(rendered)
-
-
-if __name__ == "__main__":
-    main()

tests/test_fleet_secret_rotation.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Regression coverage for timmy-home #694 fleet secret rotation assets."""
+
+from pathlib import Path
+import unittest
+
+import yaml
+
+
+ROOT = Path(__file__).resolve().parents[1]
+ANSIBLE_DIR = ROOT / "ansible"
+HOSTS_FILE = ANSIBLE_DIR / "inventory" / "hosts.ini"
+TARGETS_FILE = ANSIBLE_DIR / "inventory" / "group_vars" / "fleet.yml"
+SECRETS_FILE = ANSIBLE_DIR / "inventory" / "group_vars" / "fleet_secrets.vault.yml"
+PLAYBOOK_FILE = ANSIBLE_DIR / "playbooks" / "rotate_fleet_secrets.yml"
+DOC_FILE = ROOT / "docs" / "FLEET_SECRET_ROTATION.md"
+
+
+class TestFleetSecretRotation(unittest.TestCase):
+    def test_inventory_declares_each_host_target(self):
+        self.assertTrue(HOSTS_FILE.exists(), "missing ansible inventory hosts file")
+        self.assertTrue(TARGETS_FILE.exists(), "missing fleet target metadata")
+
+        hosts_text = HOSTS_FILE.read_text(encoding="utf-8")
+        self.assertIn("[fleet]", hosts_text)
+        self.assertIn("ezra", hosts_text)
+        self.assertIn("bezalel", hosts_text)
+
+        targets = yaml.safe_load(TARGETS_FILE.read_text(encoding="utf-8"))
+        self.assertIn("fleet_secret_targets", targets)
+
+        expected_env_files = {
+            "ezra": "/root/wizards/ezra/home/.env",
+            "bezalel": "/root/wizards/bezalel/home/.env",
+        }
+        for host, env_file in expected_env_files.items():
+            self.assertIn(host, targets["fleet_secret_targets"])
+            target = targets["fleet_secret_targets"][host]
+            self.assertEqual(target["env_file"], env_file)
+            self.assertEqual(target["ssh_authorized_keys_file"], "/root/.ssh/authorized_keys")
+            self.assertGreaterEqual(len(target["services"]), 1)
+            self.assertGreaterEqual(len(target["required_env_keys"]), 3)
+
+    def test_vault_file_contains_encrypted_secret_bundle_for_each_host(self):
+        self.assertTrue(SECRETS_FILE.exists(), "missing vaulted secrets inventory")
+        text = SECRETS_FILE.read_text(encoding="utf-8")
+        self.assertIn("fleet_secret_bundle:", text)
+        self.assertIn("$ANSIBLE_VAULT;1.1;AES256", text)
+        for host in ("ezra", "bezalel"):
+            self.assertIn(f"  {host}:", text)
+        self.assertGreaterEqual(text.count("!vault |"), 4)
+
+    def test_playbook_has_staging_verification_and_rollback(self):
+        self.assertTrue(PLAYBOOK_FILE.exists(), "missing rotation playbook")
+        text = PLAYBOOK_FILE.read_text(encoding="utf-8")
+        for snippet in (
+            "any_errors_fatal: true",
+            "vars_files:",
+            "fleet_secrets.vault.yml",
+            "backup_root",
+            "env_backup_path",
+            "ssh_backup_path",
+            "lineinfile:",
+            "copy:",
+            "systemd:",
+            "state: restarted",
+            "systemctl is-active",
+            "block:",
+            "rescue:",
+        ):
+            self.assertIn(snippet, text)
+
+    def test_docs_explain_rotation_command_and_rollback(self):
+        self.assertTrue(DOC_FILE.exists(), "missing fleet secret rotation docs")
+        text = DOC_FILE.read_text(encoding="utf-8")
+        for snippet in (
+            "ansible-playbook",
+            "--ask-vault-pass",
+            "rollback",
+            "authorized_keys",
+            "fleet_secret_bundle",
+        ):
+            self.assertIn(snippet, text)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

tests/test_lab_007_grid_power_packet.py

@@ -1,69 +0,0 @@
-from pathlib import Path
-import importlib.util
-import unittest
-
-
-ROOT = Path(__file__).resolve().parents[1]
-SCRIPT_PATH = ROOT / "scripts" / "lab_007_grid_power_packet.py"
-DOC_PATH = ROOT / "docs" / "LAB_007_GRID_POWER_REQUEST.md"
-
-
-def load_module(path: Path, name: str):
-    assert path.exists(), f"missing {path.relative_to(ROOT)}"
-    spec = importlib.util.spec_from_file_location(name, path)
-    assert spec and spec.loader
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
-class TestLab007GridPowerPacket(unittest.TestCase):
-    def test_packet_marks_missing_site_fields_and_uses_eversource_as_primary(self):
-        mod = load_module(SCRIPT_PATH, "lab_007_grid_power_packet")
-        packet = mod.build_packet({})
-
-        self.assertEqual(packet["primary_utility"]["name"], "Eversource")
-        self.assertIn("Lempster", packet["territory_evidence"])
-        self.assertFalse(packet["ready_to_contact"])
-        self.assertIn("site_address", packet["missing_fields"])
-        self.assertIn("pole_distance_feet", packet["missing_fields"])
-        self.assertIn("terrain_description", packet["missing_fields"])
-
-    def test_packet_with_site_details_builds_contact_ready_email_and_call_checklist(self):
-        mod = load_module(SCRIPT_PATH, "lab_007_grid_power_packet")
-        packet = mod.build_packet(
-            {
-                "site_address": "123 Example Rd, Lempster, NH",
-                "pole_distance_feet": 280,
-                "terrain_description": "mixed woods, uphill grade, likely overhead run",
-                "service_size": "200A residential service",
-            }
-        )
-
-        self.assertTrue(packet["ready_to_contact"])
-        self.assertEqual(packet["missing_fields"], [])
-        self.assertIn("123 Example Rd", packet["email_body"])
-        self.assertIn("280 feet", packet["email_body"])
-        self.assertIn("meter base", packet["email_body"])
-        self.assertIn("per-kWh rate", packet["estimate_request_checklist"])
-        self.assertIn("800-362-7764", packet["primary_utility"]["phone"])
-        self.assertIn("nhnewservice@eversource.com", packet["primary_utility"]["email"])
-
-    def test_repo_contains_grounded_request_doc(self):
-        self.assertTrue(DOC_PATH.exists(), "missing committed LAB-007 request doc")
-        text = DOC_PATH.read_text(encoding="utf-8")
-        for snippet in (
-            "# LAB-007 — Grid Power Hookup Estimate Request Packet",
-            "Eversource",
-            "800-362-7764",
-            "nhnewservice@eversource.com",
-            "No formal estimate has been received yet.",
-            "pole/transformer",
-            "monthly base charge",
-            "per-kWh rate",
-        ):
-            self.assertIn(snippet, text)
-
-
-if __name__ == "__main__":
-    unittest.main()