Merge pull request 'Encrypt synchronized unfiled drafts at rest' (#1099) from timmy/1098-encrypt-unfiled-drafts into main

README.md

@@ -191,8 +191,10 @@ cannot alter a newer crash-recovery claim. Device purge cancels an active drain
 private outbox storage. Results are coordinated through a bounded SQLite ledger. All private SQLite stores enforce a
 filesystem boundary independently of the service umask: the database directory is repaired to
 owner-only `0700`, database and SQLite sidecar files are owner-only `0600`, and symlinked database
-paths are rejected before access. This protects state from unrelated local accounts, but it is not
-encryption at rest; secure host access, encrypted volumes, and private backups are still required.
+paths are rejected before access. Synchronized unfiled Draft collections add AES-256-GCM payload
+encryption with account and revision authentication; existing plaintext rows migrate on their first
+read without changing revision or order. Other private stores are not encrypted at the application
+layer, so secure host access, encrypted volumes, and private backups are still required.
 Set `STACKCHAIN_STATE_DIR` to a
 persistent, writable service directory (or set `STACKCHAIN_IDEMPOTENCY_DB` to an explicit
 SQLite path); the local default is `.stackchain-state/idempotency.sqlite3`. Ledger reads and
@@ -258,6 +260,13 @@ export STACKCHAIN_PASSKEY_MAX_CHALLENGES_PER_SOURCE=10
 export STACKCHAIN_PASSKEY_MAX_CHALLENGES=10000
 # Optional; defaults to STACKCHAIN_STATE_DIR/login-attempts.sqlite3.
 export STACKCHAIN_LOGIN_ATTEMPT_DB='/var/lib/stackchain-dashboard/login-attempts.sqlite3'
+# Required for cross-device unfiled Draft sync. Inject the base64 encoding of
+# exactly 32 random bytes from a secret manager; never commit the value.
+# Generate once, for example: openssl rand -base64 32
+export STACKCHAIN_UNFILED_DRAFT_ENCRYPTION_KEY='<base64-encoded-32-byte-key>'
+# Keep this key stable and back it up separately from the database. There is no
+# online key rotation: replacing or losing it makes existing encrypted Drafts
+# unavailable. Restore the prior key to recover them before planning a rotation.
 # Trust forwarding headers only from these immediate reverse-proxy networks.
 export STACKCHAIN_TRUSTED_PROXY_CIDRS='127.0.0.0/8'
 # Optional Web Push. Generate a VAPID key pair outside the repo and inject it.

requirements.txt

@@ -1,4 +1,5 @@
 fastapi==0.133.1
+cryptography==50.0.0
 httpx==0.28.1
 pydantic==2.13.4
 Pillow==12.3.0

src/main.py

@@ -60,7 +60,12 @@ from src.push_endpoint_policy import UnsafePushEndpoint, validate_public_push_en
 from src.push_subscription_store import PushSubscriptionStore
 from src.request_boundary import RequestBodyLimitMiddleware, request_body_limit
 from src.saved_search_store import SavedSearchConflict, SavedSearchStore
-from src.unfiled_draft_store import UnfiledDraftConflict, UnfiledDraftStore
+from src.unfiled_draft_store import (
+    UnfiledDraftConflict,
+    UnfiledDraftEncryptionError,
+    UnfiledDraftStore,
+    decode_unfiled_draft_encryption_key,
+)
 from src.security_event_store import SecurityEventStore, SecurityEventStoreError
 from src.suggestion_engine import compute
 from src.later_store import LaterStore
@@ -2285,7 +2290,10 @@ def _completed_filed_review_store() -> CompletedFiledReviewStore:
 
 def _unfiled_draft_store() -> UnfiledDraftStore:
     return UnfiledDraftStore(
-        os.getenv("STACKCHAIN_UNFILED_DRAFT_DB", str(_state_dir / "unfiled-drafts.sqlite3"))
+        os.getenv("STACKCHAIN_UNFILED_DRAFT_DB", str(_state_dir / "unfiled-drafts.sqlite3")),
+        encryption_key=decode_unfiled_draft_encryption_key(
+            os.getenv("STACKCHAIN_UNFILED_DRAFT_ENCRYPTION_KEY", "")
+        ),
     )
 
 
@@ -2386,7 +2394,7 @@ async def get_unfiled_drafts(response: Response):
     login = await _confirmed_login()
     try:
         snapshot = await asyncio.to_thread(_unfiled_draft_store().get, login)
-    except (OSError, sqlite3.Error):
+    except (OSError, sqlite3.Error, UnfiledDraftEncryptionError):
         raise HTTPException(
             status_code=503,
             detail="Draft synchronization is unavailable",
@@ -2416,7 +2424,7 @@ async def replace_unfiled_drafts(payload: UnfiledDraftCollection):
         )
     except ValueError as exc:
         raise HTTPException(status_code=422, detail=str(exc))
-    except (OSError, sqlite3.Error):
+    except (OSError, sqlite3.Error, UnfiledDraftEncryptionError):
         raise HTTPException(
             status_code=503,
             detail="Draft synchronization is unavailable",

src/unfiled_draft_store.py

@@ -3,11 +3,15 @@
 import base64
 import binascii
 import json
+import os
 import re
 import sqlite3
 from datetime import datetime
 from pathlib import Path
 
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
 from src.private_state import connect_private_sqlite
 
 
@@ -24,16 +28,41 @@ class UnfiledDraftConflict(ValueError):
         self.snapshot = snapshot
 
 
+class UnfiledDraftEncryptionError(RuntimeError):
+    """Raised when private draft state cannot be authenticated and decrypted."""
+
+
+def decode_unfiled_draft_encryption_key(encoded: str) -> bytes:
+    """Decode the independently injected 256-bit draft-encryption key."""
+    try:
+        key = base64.b64decode(encoded, validate=True)
+    except (binascii.Error, ValueError) as error:
+        raise UnfiledDraftEncryptionError(
+            "unfiled draft encryption key is invalid"
+        ) from error
+    if len(key) != 32:
+        raise UnfiledDraftEncryptionError(
+            "unfiled draft encryption key must decode to exactly 32 bytes"
+        )
+    return key
+
+
 class UnfiledDraftStore:
     def __init__(
         self,
         path: str | Path,
         *,
+        encryption_key: bytes,
         limit: int = 20,
         max_total_bytes: int = 12 * 1024 * 1024,
         timeout: float = 1.0,
     ):
         self.path = Path(path)
+        if not isinstance(encryption_key, bytes) or len(encryption_key) != 32:
+            raise UnfiledDraftEncryptionError(
+                "unfiled draft encryption requires exactly 32 key bytes"
+            )
+        self._cipher = AESGCM(encryption_key)
         self.limit = limit
         self.max_total_bytes = max_total_bytes
         self.timeout = timeout
@@ -61,18 +90,71 @@ class UnfiledDraftStore:
         return normalized
 
     @staticmethod
-    def _snapshot(row) -> dict:
-        return {"revision": 0, "drafts": []} if row is None else {
-            "revision": int(row[0]), "drafts": json.loads(row[1])
-        }
+    def _aad(login: str, revision: int) -> bytes:
+        return f"stackchain:unfiled-drafts:v1\0{login}\0{revision}".encode()
+
+    def _encrypt(self, login: str, revision: int, drafts: list[dict]) -> str:
+        plaintext = json.dumps(drafts, separators=(",", ":")).encode()
+        nonce = os.urandom(12)
+        sealed = nonce + self._cipher.encrypt(
+            nonce, plaintext, self._aad(login, revision)
+        )
+        return "v1:" + base64.urlsafe_b64encode(sealed).decode()
+
+    def _decrypt(self, login: str, revision: int, payload: str) -> list[dict]:
+        try:
+            sealed = base64.b64decode(payload[3:], altchars=b"-_", validate=True)
+            if len(sealed) < 12 + 16:
+                raise ValueError("encrypted payload is too short")
+            plaintext = self._cipher.decrypt(
+                sealed[:12], sealed[12:], self._aad(login, revision)
+            )
+            drafts = json.loads(plaintext)
+            if not isinstance(drafts, list):
+                raise ValueError("decrypted payload is not a collection")
+            return drafts
+        except (binascii.Error, InvalidTag, UnicodeDecodeError, ValueError, json.JSONDecodeError) as error:
+            raise UnfiledDraftEncryptionError(
+                "unfiled drafts could not be decrypted"
+            ) from error
+
+    def _snapshot(self, login: str, row) -> tuple[dict, bool]:
+        if row is None:
+            return {"revision": 0, "drafts": []}, False
+        revision, payload = int(row[0]), row[1]
+        if payload.startswith("v1:"):
+            drafts = self._decrypt(login, revision, payload)
+            return {"revision": revision, "drafts": drafts}, False
+        try:
+            drafts = json.loads(payload)
+        except (TypeError, json.JSONDecodeError) as error:
+            raise UnfiledDraftEncryptionError(
+                "legacy unfiled drafts could not be decoded"
+            ) from error
+        if not isinstance(drafts, list):
+            raise UnfiledDraftEncryptionError(
+                "legacy unfiled drafts could not be decoded"
+            )
+        return {"revision": revision, "drafts": drafts}, True
 
     def get(self, login: str) -> dict:
+        login = self._login(login)
         with self._connect() as connection:
+            connection.execute("BEGIN IMMEDIATE")
             row = connection.execute(
                 "SELECT revision, drafts FROM unfiled_drafts WHERE login = ?",
-                (self._login(login),),
+                (login,),
             ).fetchone()
-        return self._snapshot(row)
+            snapshot, legacy = self._snapshot(login, row)
+            if legacy:
+                connection.execute(
+                    "UPDATE unfiled_drafts SET drafts = ? WHERE login = ?",
+                    (
+                        self._encrypt(login, snapshot["revision"], snapshot["drafts"]),
+                        login,
+                    ),
+                )
+        return snapshot
 
     @staticmethod
     def _filing_plan(raw: object) -> dict | None:
@@ -233,16 +315,16 @@ class UnfiledDraftStore:
         if not isinstance(expected_revision, int) or isinstance(expected_revision, bool) or expected_revision < 0:
             raise ValueError("revision is invalid")
         normalized = self._normalize(drafts)
-        serialized = json.dumps(normalized, separators=(",", ":"))
         with self._connect() as connection:
             connection.execute("BEGIN IMMEDIATE")
             row = connection.execute(
                 "SELECT revision, drafts FROM unfiled_drafts WHERE login = ?", (login,)
             ).fetchone()
-            current = self._snapshot(row)
+            current, _legacy = self._snapshot(login, row)
             if current["revision"] != expected_revision:
                 raise UnfiledDraftConflict(current)
             revision = expected_revision + 1
+            serialized = self._encrypt(login, revision, normalized)
             connection.execute(
                 "INSERT INTO unfiled_drafts(login, revision, drafts) VALUES (?, ?, ?) "
                 "ON CONFLICT(login) DO UPDATE SET revision=excluded.revision, drafts=excluded.drafts",

tests/e2e/test_mobile_offline_issue_release.py

@@ -84,6 +84,9 @@ def release_server(archive: Path, tmp_path: Path, gitea_url: str):
         "STACKCHAIN_DASHBOARD_AUTH_MODE": "operator",
         "STACKCHAIN_DASHBOARD_ACCESS_TOKEN": ACCESS_TOKEN,
         "STACKCHAIN_DASHBOARD_SESSION_SECRET": "artifact-browser-independent-session-secret-863",
+        "STACKCHAIN_UNFILED_DRAFT_ENCRYPTION_KEY": (
+            "ZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGQ="
+        ),
         "STACKCHAIN_DASHBOARD_PUBLIC_ORIGIN": origin,
         "STACKCHAIN_STATE_DIR": str(tmp_path / "state"),
     }

tests/test_private_state.py

@@ -84,7 +84,7 @@ def test_private_sqlite_connection_rejects_a_symlink_before_opening_target(tmp_p
         TodayStore,
         LaterStore,
         SavedSearchStore,
-        UnfiledDraftStore,
+        lambda path: UnfiledDraftStore(path, encryption_key=b"d" * 32),
         CompletedFiledReviewStore,
         PushSubscriptionStore,
         LiveSnapshotStore,

tests/test_unfiled_draft_store.py

@@ -1,10 +1,16 @@
+import json
+import sqlite3
+
 import httpx
 import pytest
 
-from src import main
+from src import main, unfiled_draft_store
 from src.unfiled_draft_store import UnfiledDraftConflict, UnfiledDraftStore
 
 
+ENCRYPTION_KEY = b"d" * 32
+
+
 def draft(draft_id="phone-capture", *, title="Broken checkout", evidence=None):
     return {
         "id": draft_id,
@@ -23,8 +29,77 @@ def draft(draft_id="phone-capture", *, title="Broken checkout", evidence=None):
     }
 
 
+def test_unfiled_drafts_encrypt_private_content_and_authenticate_the_account(tmp_path):
+    path = tmp_path / "unfiled.sqlite3"
+    store = UnfiledDraftStore(path, encryption_key=ENCRYPTION_KEY)
+
+    created = store.replace("timmy", 0, [draft()])
+
+    with sqlite3.connect(path) as connection:
+        payload = connection.execute(
+            "SELECT drafts FROM unfiled_drafts WHERE login = 'timmy'"
+        ).fetchone()[0]
+        connection.execute(
+            "INSERT INTO unfiled_drafts(login, revision, drafts) VALUES (?, ?, ?)",
+            ("alexander", 1, payload),
+        )
+    assert payload.startswith("v1:")
+    assert "Broken checkout" not in payload
+    assert "Steps from the field" not in payload
+    assert "cG5nLWJ5dGVz" not in payload
+    assert store.get("timmy") == created
+    with pytest.raises(
+        unfiled_draft_store.UnfiledDraftEncryptionError,
+        match="could not be decrypted",
+    ):
+        store.get("alexander")
+
+
+def test_unfiled_drafts_migrate_plaintext_without_changing_revision_or_order(tmp_path):
+    path = tmp_path / "unfiled.sqlite3"
+    store = UnfiledDraftStore(path, encryption_key=ENCRYPTION_KEY)
+    legacy = [draft(), draft("second", title="Second")]
+    with sqlite3.connect(path) as connection:
+        connection.execute(
+            "INSERT INTO unfiled_drafts(login, revision, drafts) VALUES (?, ?, ?)",
+            ("timmy", 7, json.dumps(legacy, separators=(",", ":"))),
+        )
+
+    assert store.get("timmy") == {"revision": 7, "drafts": legacy}
+    with sqlite3.connect(path) as connection:
+        migrated = connection.execute(
+            "SELECT revision, drafts FROM unfiled_drafts WHERE login = 'timmy'"
+        ).fetchone()
+    assert migrated[0] == 7
+    assert migrated[1].startswith("v1:")
+    assert "Broken checkout" not in migrated[1]
+
+
+def test_unfiled_drafts_reject_tampered_ciphertext(tmp_path):
+    path = tmp_path / "unfiled.sqlite3"
+    store = UnfiledDraftStore(path, encryption_key=ENCRYPTION_KEY)
+    store.replace("timmy", 0, [draft()])
+    with sqlite3.connect(path) as connection:
+        payload = connection.execute(
+            "SELECT drafts FROM unfiled_drafts WHERE login = 'timmy'"
+        ).fetchone()[0]
+        replacement = "A" if payload[-1] != "A" else "B"
+        connection.execute(
+            "UPDATE unfiled_drafts SET drafts = ? WHERE login = 'timmy'",
+            (payload[:-1] + replacement,),
+        )
+
+    with pytest.raises(
+        unfiled_draft_store.UnfiledDraftEncryptionError,
+        match="could not be decrypted",
+    ):
+        store.get("timmy")
+
+
 def test_unfiled_drafts_are_revisioned_ordered_and_account_scoped(tmp_path):
-    store = UnfiledDraftStore(tmp_path / "unfiled.sqlite3")
+    store = UnfiledDraftStore(
+        tmp_path / "unfiled.sqlite3", encryption_key=ENCRYPTION_KEY
+    )
 
     created = store.replace(" Timmy ", 0, [draft(), draft("second", title="Second")])
 
@@ -37,7 +112,12 @@ def test_unfiled_drafts_are_revisioned_ordered_and_account_scoped(tmp_path):
 
 
 def test_unfiled_drafts_bound_collection_and_decoded_evidence(tmp_path):
-    store = UnfiledDraftStore(tmp_path / "unfiled.sqlite3", limit=2, max_total_bytes=12)
+    store = UnfiledDraftStore(
+        tmp_path / "unfiled.sqlite3",
+        encryption_key=ENCRYPTION_KEY,
+        limit=2,
+        max_total_bytes=12,
+    )
 
     with pytest.raises(ValueError, match="limited to 2"):
         store.replace("timmy", 0, [draft("one", evidence=[]), draft("two", evidence=[]), draft("three", evidence=[])])
@@ -52,7 +132,9 @@ def test_unfiled_drafts_bound_collection_and_decoded_evidence(tmp_path):
 
 
 def test_unfiled_drafts_allow_untitled_photo_evidence_but_reject_empty_records(tmp_path):
-    store = UnfiledDraftStore(tmp_path / "unfiled.sqlite3")
+    store = UnfiledDraftStore(
+        tmp_path / "unfiled.sqlite3", encryption_key=ENCRYPTION_KEY
+    )
     photo_only = draft(title="")
 
     validated = main.UnfiledDraft.model_validate(photo_only).model_dump()
@@ -66,7 +148,9 @@ def test_unfiled_drafts_allow_untitled_photo_evidence_but_reject_empty_records(t
 
 
 def test_unfiled_drafts_validate_and_round_trip_complete_filing_plan(tmp_path):
-    store = UnfiledDraftStore(tmp_path / "unfiled.sqlite3")
+    store = UnfiledDraftStore(
+        tmp_path / "unfiled.sqlite3", encryption_key=ENCRYPTION_KEY
+    )
     planned = draft(evidence=[])
     planned.pop("evidence")
     planned["filing_plan"] = {
@@ -102,6 +186,10 @@ async def test_unfiled_draft_api_is_account_scoped_csrf_protected_and_conflict_s
     monkeypatch.setenv("STACKCHAIN_SESSION_DB", str(tmp_path / "sessions.sqlite3"))
     monkeypatch.setenv("STACKCHAIN_LOGIN_ATTEMPT_DB", str(tmp_path / "login.sqlite3"))
     monkeypatch.setenv("STACKCHAIN_UNFILED_DRAFT_DB", str(tmp_path / "drafts.sqlite3"))
+    monkeypatch.setenv(
+        "STACKCHAIN_UNFILED_DRAFT_ENCRYPTION_KEY",
+        "ZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGQ=",
+    )
     identity = {"login": "Timmy"}
 
     async def user():
@@ -133,6 +221,10 @@ async def test_unfiled_draft_api_is_account_scoped_csrf_protected_and_conflict_s
         fetched = await client.get("/api/v1/unfiled-drafts")
         identity["login"] = "Alexander"
         isolated = await client.get("/api/v1/unfiled-drafts")
+        monkeypatch.delenv("STACKCHAIN_UNFILED_DRAFT_ENCRYPTION_KEY")
+        unavailable = await client.get("/api/v1/unfiled-drafts")
+        monkeypatch.setenv("STACKCHAIN_UNFILED_DRAFT_ENCRYPTION_KEY", "not-base64")
+        malformed = await client.get("/api/v1/unfiled-drafts")
 
     assert forbidden.status_code == 403
     assert saved.status_code == 200
@@ -144,3 +236,7 @@ async def test_unfiled_draft_api_is_account_scoped_csrf_protected_and_conflict_s
     assert fetched.json() == saved.json()
     assert fetched.headers["cache-control"] == "no-store"
     assert isolated.json() == {"revision": 0, "drafts": []}
+    assert unavailable.status_code == 503
+    assert unavailable.json()["detail"] == "Draft synchronization is unavailable"
+    assert malformed.status_code == 503
+    assert malformed.json()["detail"] == "Draft synchronization is unavailable"