Merge pull request 'Encrypt Security activity journal at rest' (#1119) from timmy/1118-encrypt-security-activity into main

README.md

@@ -267,12 +267,13 @@ export STACKCHAIN_PASSKEY_MAX_CHALLENGES_PER_SOURCE=10
 export STACKCHAIN_PASSKEY_MAX_CHALLENGES=10000
 # Optional; defaults to STACKCHAIN_STATE_DIR/login-attempts.sqlite3.
 export STACKCHAIN_LOGIN_ATTEMPT_DB='/var/lib/stackchain-dashboard/login-attempts.sqlite3'
-# Required for worker-shared live/Find Work snapshots and synchronized Today/Later
-# planning state. Keep this key independent from the Draft key and inject the
+# Required for worker-shared live/Find Work snapshots, synchronized Today/Later
+# planning state, and the Security activity journal. Keep this key independent
+# from the Draft key and inject the
 # base64 encoding of exactly 32 random bytes from a secret manager. Never commit
 # it. Missing, malformed, wrong-key, or modified state fails closed without
-# returning content. Legacy Today/Later rows migrate atomically on first use
-# without advancing their logical revision.
+# returning content. Legacy Today/Later and Security activity rows migrate
+# atomically on first use without changing logical revisions or journal IDs.
 export STACKCHAIN_PRIVATE_STATE_ENCRYPTION_KEY='<base64-encoded-32-byte-key>'
 # Required for cross-device unfiled Draft sync. The single-key setting remains
 # supported for the first deployment of keyring-capable code and writes v1 envelopes.
@@ -386,8 +387,13 @@ place, their live sessions remain valid, and their idle clock starts at migratio
 The same sheet includes **Security activity**, a reverse-chronological journal of
 successful token/passkey sign-ins, passkey enrollments, sign-outs, remote device
 revocations, issue closures, and pull-request merges. The separate SQLite journal
-retains at most 10,000 events for 90 days and stores only bounded device labels and
-action targets. It never stores access tokens, cookies, session/CSRF values,
+retains at most 10,000 events for 90 days. Its bounded event kind, authentication
+method, device label, and action target are sealed at rest with the private-state
+encryption key and authenticated against the immutable event ID. Legacy plaintext
+journals migrate atomically without changing IDs, timestamps, order, or pending
+operations. Missing or wrong key material and modified ciphertext fail closed with
+no partial activity response. The journal never stores access tokens, cookies,
+session/CSRF values,
 credential IDs, public keys, challenges, attestation data, raw network addresses,
 request bodies, or comment content. Keep its database on the same class of persistent,
 writable storage as the session registry. Before passkey credential creation, session

src/security_event_store.py

@@ -7,6 +7,11 @@ from pathlib import Path
 from typing import Callable
 
 from src.private_state import connect_private_sqlite
+from src.state_encryption import (
+    PrivateStateCipher,
+    PrivateStateEncryptionError,
+    private_state_encryption_key,
+)
 
 
 class SecurityEventStoreError(RuntimeError):
@@ -39,12 +44,22 @@ class SecurityEventStore:
         max_events: int = 10_000,
         retention_seconds: int = 90 * 24 * 60 * 60,
         lock_timeout_seconds: float = 0.1,
+        encryption_key: bytes | None = None,
     ) -> None:
         self.path = Path(path)
         self.clock = clock
         self.max_events = max(1, max_events)
         self.retention_seconds = max(1, retention_seconds)
         self.lock_timeout_seconds = lock_timeout_seconds
+        try:
+            self._cipher = PrivateStateCipher(
+                encryption_key if encryption_key is not None else private_state_encryption_key(),
+                store="security-events",
+            )
+        except PrivateStateEncryptionError as exc:
+            raise SecurityEventStoreError(
+                "Security activity is temporarily unavailable"
+            ) from exc
 
     @staticmethod
     def _bounded(value: str | None, limit: int) -> str | None:
@@ -60,10 +75,7 @@ class SecurityEventStore:
                 """
                 CREATE TABLE IF NOT EXISTS security_events (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
-                    kind TEXT NOT NULL,
-                    method TEXT,
-                    device_label TEXT,
-                    target TEXT,
+                    payload TEXT,
                     created_at INTEGER NOT NULL,
                     status TEXT NOT NULL DEFAULT 'completed',
                     operation_id TEXT
@@ -81,6 +93,11 @@ class SecurityEventStore:
                 connection.execute(
                     "ALTER TABLE security_events ADD COLUMN operation_id TEXT"
                 )
+            plaintext_columns = {"kind", "method", "device_label", "target"}
+            if "payload" not in columns or plaintext_columns.intersection(columns):
+                self._migrate_plaintext(
+                    connection, has_payload="payload" in columns
+                )
             connection.execute(
                 "CREATE INDEX IF NOT EXISTS security_events_created "
                 "ON security_events(created_at DESC, id DESC)"
@@ -95,6 +112,56 @@ class SecurityEventStore:
                 "Security activity is temporarily unavailable"
             ) from exc
 
+    def _migrate_plaintext(
+        self, connection: sqlite3.Connection, *, has_payload: bool
+    ) -> None:
+        connection.execute("PRAGMA secure_delete = ON")
+        payload_column = "payload," if has_payload else "NULL AS payload,"
+        rows = connection.execute(
+            f"SELECT id, {payload_column} kind, method, device_label, target, "
+            "created_at, status, operation_id FROM security_events ORDER BY id"
+        ).fetchall()
+        connection.execute(
+            """
+            CREATE TABLE security_events_encrypted (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                payload TEXT,
+                created_at INTEGER NOT NULL,
+                status TEXT NOT NULL DEFAULT 'completed',
+                operation_id TEXT
+            )
+            """
+        )
+        for (
+            event_id,
+            payload,
+            kind,
+            method,
+            device_label,
+            target,
+            created_at,
+            status,
+            operation_id,
+        ) in rows:
+            connection.execute(
+                "INSERT INTO security_events_encrypted "
+                "(id, payload, created_at, status, operation_id) VALUES (?, ?, ?, ?, ?)",
+                (
+                    event_id,
+                    payload
+                    or self._seal_event(
+                        event_id, kind, method, device_label, target
+                    ),
+                    created_at,
+                    status,
+                    operation_id,
+                ),
+            )
+        connection.execute("DROP TABLE security_events")
+        connection.execute(
+            "ALTER TABLE security_events_encrypted RENAME TO security_events"
+        )
+
     def _prune(self, connection: sqlite3.Connection, now: int) -> None:
         connection.execute(
             "DELETE FROM security_events WHERE created_at < ?",
@@ -106,6 +173,24 @@ class SecurityEventStore:
             (self.max_events,),
         )
 
+    def _seal_event(
+        self,
+        event_id: int,
+        kind: str,
+        method: str | None,
+        device_label: str | None,
+        target: str | None,
+    ) -> str:
+        return self._cipher.seal(
+            {
+                "kind": self._bounded(kind, 48) or "security_event",
+                "method": self._bounded(method, 32),
+                "device_label": self._bounded(device_label, 64),
+                "target": self._bounded(target, 255),
+            },
+            binding=f"event:{event_id}",
+        )
+
     def record(
         self,
         kind: str,
@@ -117,16 +202,17 @@ class SecurityEventStore:
         now = int(self.clock())
         try:
             with self._connect() as connection:
+                cursor = connection.execute(
+                    "INSERT INTO security_events(created_at, status) VALUES (?, 'completed')",
+                    (now,),
+                )
+                event_id = cursor.lastrowid
+                payload = self._seal_event(
+                    event_id, kind, method, device_label, target
+                )
                 connection.execute(
-                    "INSERT INTO security_events(kind, method, device_label, target, created_at, status) "
-                    "VALUES (?, ?, ?, ?, ?, 'completed')",
-                    (
-                        self._bounded(kind, 48) or "security_event",
-                        self._bounded(method, 32),
-                        self._bounded(device_label, 64),
-                        self._bounded(target, 255),
-                        now,
-                    ),
+                    "UPDATE security_events SET payload = ? WHERE id = ?",
+                    (payload, event_id),
                 )
                 self._prune(connection, now)
         except (OSError, sqlite3.Error) as exc:
@@ -146,16 +232,19 @@ class SecurityEventStore:
         operation_id = secrets.token_urlsafe(24)
         try:
             with self._connect() as connection:
+                cursor = connection.execute(
+                    "INSERT INTO security_events(created_at, status, operation_id) "
+                    "VALUES (?, 'pending', ?)",
+                    (now, operation_id),
+                )
+                event_id = cursor.lastrowid
                 connection.execute(
-                    "INSERT INTO security_events(kind, method, device_label, target, created_at, status, operation_id) "
-                    "VALUES (?, ?, ?, ?, ?, 'pending', ?)",
+                    "UPDATE security_events SET payload = ? WHERE id = ?",
                     (
-                        self._bounded(kind, 48) or "security_event",
-                        self._bounded(method, 32),
-                        self._bounded(device_label, 64),
-                        self._bounded(target, 255),
-                        now,
-                        operation_id,
+                        self._seal_event(
+                            event_id, kind, method, device_label, target
+                        ),
+                        event_id,
                     ),
                 )
                 self._prune(connection, now)
@@ -204,7 +293,7 @@ class SecurityEventStore:
         try:
             with self._connect() as connection:
                 rows = connection.execute(
-                    "SELECT id, kind, method, device_label, target, created_at, status "
+                    "SELECT id, payload, created_at, status "
                     f"FROM security_events {where} ORDER BY id DESC LIMIT ?",
                     parameters,
                 ).fetchall()
@@ -214,7 +303,32 @@ class SecurityEventStore:
             ) from exc
         has_more = len(rows) > bounded_limit
         visible = rows[:bounded_limit]
+        events = []
+        try:
+            for event_id, payload, created_at, status in visible:
+                value, _legacy = self._cipher.open(
+                    payload, binding=f"event:{event_id}"
+                )
+                if not isinstance(value, dict) or not isinstance(value.get("kind"), str):
+                    raise PrivateStateEncryptionError(
+                        "private state could not be decrypted"
+                    )
+                events.append(
+                    SecurityEvent(
+                        event_id,
+                        value["kind"],
+                        value.get("method"),
+                        value.get("device_label"),
+                        value.get("target"),
+                        created_at,
+                        status,
+                    )
+                )
+        except PrivateStateEncryptionError as exc:
+            raise SecurityEventStoreError(
+                "Security activity is temporarily unavailable"
+            ) from exc
         return SecurityEventPage(
-            events=[SecurityEvent(*row) for row in visible],
+            events=events,
             next_cursor=visible[-1][0] if has_more else None,
         )

tests/e2e/test_mobile_today_handoff_release.py

@@ -530,11 +530,20 @@ def test_release_artifact_recovers_admitted_blocker_after_reload_and_opens_next_
             expect(page.locator("#resume-today-break")).to_be_visible()
 
             page.evaluate("""() => {
+              const outbox = JSON.parse(localStorage.getItem('stackchain.authored-outbox.v1') ||
+                '{"version":2,"items":[]}');
+              outbox.items.push({
+                id: 'already-admitted-blocker', operationId: 'already-admitted-blocker',
+                kind: 'issue-comment', repository: 'acme/mobile', number: 41,
+                body: 'Blocked after reload waiting for the design owner', ownerLogin: 'timmy',
+                status: 'queued', queuedAt: Date.now()
+              });
+              localStorage.setItem('stackchain.authored-outbox.v1', JSON.stringify(outbox));
               localStorage.setItem('stackchain.today-progress.v1.timmy', JSON.stringify({
                 version: 1,
                 drafts: {
                   'issue:acme/mobile:41:': {
-                    body: 'Blocked waiting for the design owner',
+                    body: 'Blocked after reload waiting for the design owner',
                     operation_id: 'already-admitted-blocker',
                     blocker_pending: true,
                     blocker_until: '2000-01-02T03:04:00.000Z'
@@ -575,17 +584,22 @@ def test_release_artifact_recovers_admitted_blocker_after_reload_and_opens_next_
             assert state["today"] == ["issue:acme/mobile:42:"]
             assert state["later"]["issue:acme/mobile:41:"].startswith("2099-08-19T09:00")
             queued_comments = [
-                item for item in state["outbox"]["items"] if item["kind"] == "issue-comment"
+                item for item in state["outbox"]["items"]
+                if item["kind"] == "issue-comment"
+                and item["number"] == 41
+                and item["body"] == "Blocked after reload waiting for the design owner"
             ]
             delivered_comments = [
                 {"number": number, "body": body}
                 for number, body in fake.comments
-                if number == 41 and body == "Blocked waiting for the design owner"
+                if number == 41 and body == "Blocked after reload waiting for the design owner"
             ]
-            assert len(queued_comments) + len(delivered_comments) == 1
-            receipt = (queued_comments + delivered_comments)[0]
+            assert len(queued_comments) <= 1
+            assert len(delivered_comments) <= 1
+            assert queued_comments or delivered_comments
+            receipt = (delivered_comments or queued_comments)[0]
             assert receipt["number"] == 41
-            assert receipt["body"] == "Blocked waiting for the design owner"
+            assert receipt["body"] == "Blocked after reload waiting for the design owner"
             assert page.evaluate("document.documentElement.scrollWidth <= window.innerWidth")
             assert browser_errors == []
             browser.close()

tests/test_security_activity.py

@@ -61,6 +61,35 @@ async def test_authenticated_security_activity_lists_private_sign_in_history(sec
     assert b"stackchain_session" not in persisted
 
 
+@pytest.mark.anyio
+async def test_tampered_security_activity_returns_no_partial_history(security_access):
+    transport = httpx.ASGITransport(app=main.app, raise_app_exceptions=False)
+    async with httpx.AsyncClient(transport=transport, base_url="https://test") as client:
+        await client.post(
+            "/api/v1/session",
+            json={
+                "access_token": "correct horse battery staple",
+                "device_label": "Phone",
+            },
+        )
+        with sqlite3.connect(security_access / "security.sqlite3") as connection:
+            payload = connection.execute(
+                "SELECT payload FROM security_events WHERE id = 1"
+            ).fetchone()[0]
+            replacement = "A" if payload[-1] != "A" else "B"
+            connection.execute(
+                "UPDATE security_events SET payload = ? WHERE id = 1",
+                (payload[:-1] + replacement,),
+            )
+
+        activity = await client.get("/api/v1/security-events")
+
+    assert activity.status_code == 503
+    assert activity.json() == {
+        "detail": "Security activity is temporarily unavailable"
+    }
+
+
 @pytest.mark.anyio
 async def test_security_activity_limit_is_validated(security_access):
     transport = httpx.ASGITransport(app=main.app)

tests/test_security_event_store.py

@@ -1,6 +1,137 @@
 import sqlite3
 
-from src.security_event_store import SecurityEventStore
+import pytest
+
+from src.security_event_store import SecurityEventStore, SecurityEventStoreError
+
+
+PRIVATE_KEY = b"e" * 32
+
+
+def test_security_event_schema_exposes_no_plaintext_metadata_columns(tmp_path):
+    store = SecurityEventStore(
+        tmp_path / "security.sqlite3", clock=lambda: 1_000, encryption_key=PRIVATE_KEY
+    )
+    store.record("issue_closed", method="passkey", target="private/repo#42")
+
+    with sqlite3.connect(store.path) as connection:
+        columns = {
+            row[1]
+            for row in connection.execute("PRAGMA table_info(security_events)")
+        }
+    assert columns == {
+        "id",
+        "payload",
+        "created_at",
+        "status",
+        "operation_id",
+    }
+
+
+def test_security_event_payload_is_encrypted_at_rest_and_survives_restart(tmp_path):
+    path = tmp_path / "security.sqlite3"
+    canaries = {
+        "kind": "private_issue_closed_canary",
+        "method": "private_passkey_canary",
+        "device_label": "Private Operator Phone Canary",
+        "target": "private-org/roadmap#4242",
+    }
+    store = SecurityEventStore(path, clock=lambda: 1_000, encryption_key=PRIVATE_KEY)
+
+    store.record(**canaries)
+
+    with sqlite3.connect(path) as connection:
+        payload = connection.execute(
+            "SELECT payload FROM security_events"
+        ).fetchone()[0]
+    assert payload.startswith("v1:")
+    database_bytes = path.read_bytes()
+    assert all(value.encode() not in database_bytes for value in canaries.values())
+    reopened = SecurityEventStore(path, clock=lambda: 1_001, encryption_key=PRIVATE_KEY)
+    event = reopened.list(limit=10).events[0]
+    assert (event.kind, event.method, event.device_label, event.target) == tuple(
+        canaries.values()
+    )
+
+
+def test_legacy_security_events_migrate_without_changing_journal_semantics(tmp_path):
+    path = tmp_path / "security.sqlite3"
+    with sqlite3.connect(path) as connection:
+        connection.executescript(
+            """
+            CREATE TABLE security_events (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                kind TEXT NOT NULL,
+                method TEXT,
+                device_label TEXT,
+                target TEXT,
+                created_at INTEGER NOT NULL,
+                status TEXT NOT NULL DEFAULT 'completed',
+                operation_id TEXT
+            );
+            INSERT INTO security_events
+                (id, kind, method, device_label, target, created_at, status, operation_id)
+            VALUES
+                (7, 'legacy_sign_in_canary', 'legacy_token_canary',
+                 'Legacy Phone Canary', 'dashboard', 900, 'completed', NULL),
+                (9, 'legacy_issue_closed_canary', 'passkey',
+                 'Legacy Tablet Canary', 'private/repo#9', 901, 'pending', 'operation-9');
+            """
+        )
+
+    store = SecurityEventStore(path, clock=lambda: 1_000, encryption_key=PRIVATE_KEY)
+    page = store.list(limit=1)
+    older = store.list(limit=10, cursor=page.next_cursor)
+
+    assert [(event.id, event.kind, event.created_at, event.status) for event in page.events] == [
+        (9, "legacy_issue_closed_canary", 901, "pending")
+    ]
+    assert [(event.id, event.kind, event.created_at, event.status) for event in older.events] == [
+        (7, "legacy_sign_in_canary", 900, "completed")
+    ]
+    store.finalize("operation-9")
+    assert store.list(limit=1).events[0].status == "completed"
+    with sqlite3.connect(path) as connection:
+        rows = connection.execute(
+            "SELECT id, payload FROM security_events ORDER BY id"
+        ).fetchall()
+    assert [row[0] for row in rows] == [7, 9]
+    assert all(row[1].startswith("v1:") for row in rows)
+    database_bytes = path.read_bytes()
+    assert b"legacy_sign_in_canary" not in database_bytes
+    assert b"Legacy Tablet Canary" not in database_bytes
+
+
+def test_wrong_key_or_tampered_security_activity_fails_closed(tmp_path):
+    path = tmp_path / "security.sqlite3"
+    store = SecurityEventStore(path, clock=lambda: 1_000, encryption_key=PRIVATE_KEY)
+    store.record("issue_closed", target="private/repo#42")
+
+    with pytest.raises(SecurityEventStoreError, match="temporarily unavailable"):
+        SecurityEventStore(
+            path, clock=lambda: 1_001, encryption_key=b"x" * 32
+        ).list(limit=10)
+
+    with sqlite3.connect(path) as connection:
+        payload = connection.execute(
+            "SELECT payload FROM security_events WHERE id = 1"
+        ).fetchone()[0]
+        replacement = "A" if payload[-1] != "A" else "B"
+        connection.execute(
+            "UPDATE security_events SET payload = ? WHERE id = 1",
+            (payload[:-1] + replacement,),
+        )
+    with pytest.raises(SecurityEventStoreError, match="temporarily unavailable"):
+        store.list(limit=10)
+
+
+def test_missing_security_activity_encryption_key_fails_with_store_error(
+    monkeypatch, tmp_path
+):
+    monkeypatch.delenv("STACKCHAIN_PRIVATE_STATE_ENCRYPTION_KEY", raising=False)
+
+    with pytest.raises(SecurityEventStoreError, match="temporarily unavailable"):
+        SecurityEventStore(tmp_path / "security.sqlite3", clock=lambda: 1_000)
 
 
 def test_security_events_are_private_bounded_and_reverse_chronological(tmp_path):
@@ -31,8 +162,7 @@ def test_security_events_are_private_bounded_and_reverse_chronological(tmp_path)
         row[1] for row in sqlite3.connect(store.path).execute("PRAGMA table_info(security_events)")
     }
     assert columns == {
-        "id", "kind", "method", "device_label", "target", "created_at",
-        "status", "operation_id",
+        "id", "payload", "created_at", "status", "operation_id",
     }