# nix/checks.nix — Build-time verification tests # # Checks are Linux-only: the full Python venv (via uv2nix) includes # transitive deps like onnxruntime that lack compatible wheels on # aarch64-darwin. The package and devShell still work on macOS. { inputs, ... }: { perSystem = { pkgs, system, lib, ... }: let hermes-agent = inputs.self.packages.${system}.default; hermesVenv = pkgs.callPackage ./python.nix { inherit (inputs) uv2nix pyproject-nix pyproject-build-systems; }; configMergeScript = pkgs.callPackage ./configMergeScript.nix { }; # Auto-generated config key reference — always in sync with Python configKeys = pkgs.runCommand "hermes-config-keys" {} '' set -euo pipefail export HOME=$TMPDIR ${hermesVenv}/bin/python3 -c ' import json, sys from hermes_cli.config import DEFAULT_CONFIG def leaf_paths(d, prefix=""): paths = [] for k, v in sorted(d.items()): path = f"{prefix}.{k}" if prefix else k if isinstance(v, dict) and v: paths.extend(leaf_paths(v, path)) else: paths.append(path) return paths json.dump(sorted(leaf_paths(DEFAULT_CONFIG)), sys.stdout, indent=2) ' > $out ''; in { packages.configKeys = configKeys; checks = lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { # Verify binaries exist and are executable package-contents = pkgs.runCommand "hermes-package-contents" { } '' set -e echo "=== Checking binaries ===" test -x ${hermes-agent}/bin/hermes || (echo "FAIL: hermes binary missing"; exit 1) test -x ${hermes-agent}/bin/hermes-agent || (echo "FAIL: hermes-agent binary missing"; exit 1) echo "PASS: All binaries present" echo "=== Checking version ===" ${hermes-agent}/bin/hermes version 2>&1 | grep -qi "hermes" || (echo "FAIL: version check"; exit 1) echo "PASS: Version check" echo "=== All checks passed ===" mkdir -p $out echo "ok" > $out/result ''; # Verify every pyproject.toml [project.scripts] entry has a wrapped binary entry-points-sync = pkgs.runCommand "hermes-entry-points-sync" { } '' set -e echo "=== Checking entry points match pyproject.toml [project.scripts] ===" for bin in hermes hermes-agent hermes-acp; do test -x ${hermes-agent}/bin/$bin || (echo "FAIL: $bin binary missing from Nix package"; exit 1) echo "PASS: $bin present" done mkdir -p $out echo "ok" > $out/result ''; # Verify CLI subcommands are accessible cli-commands = pkgs.runCommand "hermes-cli-commands" { } '' set -e export HOME=$(mktemp -d) echo "=== Checking hermes --help ===" ${hermes-agent}/bin/hermes --help 2>&1 | grep -q "gateway" || (echo "FAIL: gateway subcommand missing"; exit 1) ${hermes-agent}/bin/hermes --help 2>&1 | grep -q "config" || (echo "FAIL: config subcommand missing"; exit 1) echo "PASS: All subcommands accessible" echo "=== All CLI checks passed ===" mkdir -p $out echo "ok" > $out/result ''; # Verify bundled skills are present in the package bundled-skills = pkgs.runCommand "hermes-bundled-skills" { } '' set -e echo "=== Checking bundled skills ===" test -d ${hermes-agent}/share/hermes-agent/skills || (echo "FAIL: skills directory missing"; exit 1) echo "PASS: skills directory exists" SKILL_COUNT=$(find ${hermes-agent}/share/hermes-agent/skills -name "SKILL.md" | wc -l) test "$SKILL_COUNT" -gt 0 || (echo "FAIL: no SKILL.md files found in skills directory"; exit 1) echo "PASS: $SKILL_COUNT bundled skills found" grep -q "HERMES_BUNDLED_SKILLS" ${hermes-agent}/bin/hermes || \ (echo "FAIL: HERMES_BUNDLED_SKILLS not in wrapper"; exit 1) echo "PASS: HERMES_BUNDLED_SKILLS set in wrapper" echo "=== All bundled skills checks passed ===" mkdir -p $out echo "ok" > $out/result ''; # Verify HERMES_MANAGED guard works on all mutation commands managed-guard = pkgs.runCommand "hermes-managed-guard" { } '' set -e export HOME=$(mktemp -d) check_blocked() { local label="$1" shift OUTPUT=$(HERMES_MANAGED=true "$@" 2>&1 || true) echo "$OUTPUT" | grep -q "managed by NixOS" || (echo "FAIL: $label not guarded"; echo "$OUTPUT"; exit 1) echo "PASS: $label blocked in managed mode" } echo "=== Checking HERMES_MANAGED guards ===" check_blocked "config set" ${hermes-agent}/bin/hermes config set model foo check_blocked "config edit" ${hermes-agent}/bin/hermes config edit echo "=== All guard checks passed ===" mkdir -p $out echo "ok" > $out/result ''; # ── Config merge + round-trip test ──────────────────────────────── # Tests the merge script (Nix activation behavior) across 7 # scenarios, then verifies Python's load_config() reads correctly. config-roundtrip = let # Nix settings used across scenarios nixSettings = pkgs.writeText "nix-settings.json" (builtins.toJSON { model = "test/nix-model"; toolsets = ["nix-toolset"]; terminal = { backend = "docker"; timeout = 999; }; mcp_servers = { nix-server = { command = "echo"; args = ["nix"]; }; }; }); # Pre-built YAML fixtures for each scenario fixtureB = pkgs.writeText "fixture-b.yaml" '' model: "old-model" mcp_servers: old-server: url: "http://old" ''; fixtureC = pkgs.writeText "fixture-c.yaml" '' skills: disabled: - skill-a - skill-b session_reset: mode: idle idle_minutes: 30 streaming: enabled: true fallback_model: provider: openrouter model: test-fallback ''; fixtureD = pkgs.writeText "fixture-d.yaml" '' model: "user-model" skills: disabled: - skill-x streaming: enabled: true transport: edit ''; fixtureE = pkgs.writeText "fixture-e.yaml" '' mcp_servers: user-server: url: "http://user-mcp" nix-server: command: "old-cmd" args: ["old"] ''; fixtureF = pkgs.writeText "fixture-f.yaml" '' terminal: cwd: "/user/path" custom_key: "preserved" env_passthrough: - USER_VAR ''; in pkgs.runCommand "hermes-config-roundtrip" { nativeBuildInputs = [ pkgs.jq ]; } '' set -e export HOME=$(mktemp -d) ERRORS="" fail() { ERRORS="$ERRORS\nFAIL: $1"; } # Helper: run merge then load with Python, output merged JSON merge_and_load() { local hermes_home="$1" export HERMES_HOME="$hermes_home" ${configMergeScript} ${nixSettings} "$hermes_home/config.yaml" ${hermesVenv}/bin/python3 -c ' import json, sys from hermes_cli.config import load_config json.dump(load_config(), sys.stdout, default=str) ' } # ═══════════════════════════════════════════════════════════════ # Scenario A: Fresh install — no existing config.yaml # ═══════════════════════════════════════════════════════════════ echo "=== Scenario A: Fresh install ===" A_HOME=$(mktemp -d) A_CONFIG=$(merge_and_load "$A_HOME") echo "$A_CONFIG" | jq -e '.model == "test/nix-model"' > /dev/null \ || fail "A: model not set from Nix" echo "$A_CONFIG" | jq -e '.mcp_servers."nix-server".command == "echo"' > /dev/null \ || fail "A: MCP nix-server missing" echo "PASS: Scenario A" # ═══════════════════════════════════════════════════════════════ # Scenario B: Nix keys override existing values # ═══════════════════════════════════════════════════════════════ echo "=== Scenario B: Nix overrides ===" B_HOME=$(mktemp -d) install -m 0644 ${fixtureB} "$B_HOME/config.yaml" B_CONFIG=$(merge_and_load "$B_HOME") echo "$B_CONFIG" | jq -e '.model == "test/nix-model"' > /dev/null \ || fail "B: Nix model did not override" echo "PASS: Scenario B" # ═══════════════════════════════════════════════════════════════ # Scenario C: User-only keys preserved # ═══════════════════════════════════════════════════════════════ echo "=== Scenario C: User keys preserved ===" C_HOME=$(mktemp -d) install -m 0644 ${fixtureC} "$C_HOME/config.yaml" C_CONFIG=$(merge_and_load "$C_HOME") echo "$C_CONFIG" | jq -e '.skills.disabled == ["skill-a", "skill-b"]' > /dev/null \ || fail "C: skills.disabled not preserved" echo "$C_CONFIG" | jq -e '.session_reset.mode == "idle"' > /dev/null \ || fail "C: session_reset.mode not preserved" echo "$C_CONFIG" | jq -e '.session_reset.idle_minutes == 30' > /dev/null \ || fail "C: session_reset.idle_minutes not preserved" echo "$C_CONFIG" | jq -e '.streaming.enabled == true' > /dev/null \ || fail "C: streaming.enabled not preserved" echo "$C_CONFIG" | jq -e '.fallback_model.provider == "openrouter"' > /dev/null \ || fail "C: fallback_model not preserved" echo "PASS: Scenario C" # ═══════════════════════════════════════════════════════════════ # Scenario D: Mixed — Nix wins for its keys, user keys preserved # ═══════════════════════════════════════════════════════════════ echo "=== Scenario D: Mixed merge ===" D_HOME=$(mktemp -d) install -m 0644 ${fixtureD} "$D_HOME/config.yaml" D_CONFIG=$(merge_and_load "$D_HOME") echo "$D_CONFIG" | jq -e '.model == "test/nix-model"' > /dev/null \ || fail "D: Nix model did not override user model" echo "$D_CONFIG" | jq -e '.skills.disabled == ["skill-x"]' > /dev/null \ || fail "D: user skills not preserved" echo "$D_CONFIG" | jq -e '.streaming.enabled == true' > /dev/null \ || fail "D: user streaming not preserved" echo "PASS: Scenario D" # ═══════════════════════════════════════════════════════════════ # Scenario E: MCP additive merge # ═══════════════════════════════════════════════════════════════ echo "=== Scenario E: MCP additive merge ===" E_HOME=$(mktemp -d) install -m 0644 ${fixtureE} "$E_HOME/config.yaml" E_CONFIG=$(merge_and_load "$E_HOME") echo "$E_CONFIG" | jq -e '.mcp_servers."user-server".url == "http://user-mcp"' > /dev/null \ || fail "E: user MCP server not preserved" echo "$E_CONFIG" | jq -e '.mcp_servers."nix-server".command == "echo"' > /dev/null \ || fail "E: Nix MCP server did not override same-name user server" echo "$E_CONFIG" | jq -e '.mcp_servers."nix-server".args == ["nix"]' > /dev/null \ || fail "E: Nix MCP server args wrong" echo "PASS: Scenario E" # ═══════════════════════════════════════════════════════════════ # Scenario F: Nested deep merge # ═══════════════════════════════════════════════════════════════ echo "=== Scenario F: Nested deep merge ===" F_HOME=$(mktemp -d) install -m 0644 ${fixtureF} "$F_HOME/config.yaml" F_CONFIG=$(merge_and_load "$F_HOME") echo "$F_CONFIG" | jq -e '.terminal.backend == "docker"' > /dev/null \ || fail "F: Nix terminal.backend did not override" echo "$F_CONFIG" | jq -e '.terminal.timeout == 999' > /dev/null \ || fail "F: Nix terminal.timeout did not override" echo "$F_CONFIG" | jq -e '.terminal.custom_key == "preserved"' > /dev/null \ || fail "F: terminal.custom_key not preserved" echo "$F_CONFIG" | jq -e '.terminal.cwd == "/user/path"' > /dev/null \ || fail "F: user terminal.cwd not preserved when Nix does not set it" echo "$F_CONFIG" | jq -e '.terminal.env_passthrough == ["USER_VAR"]' > /dev/null \ || fail "F: user terminal.env_passthrough not preserved" echo "PASS: Scenario F" # ═══════════════════════════════════════════════════════════════ # Scenario G: Idempotency — merging twice yields the same result # ═══════════════════════════════════════════════════════════════ echo "=== Scenario G: Idempotency ===" G_HOME=$(mktemp -d) install -m 0644 ${fixtureD} "$G_HOME/config.yaml" ${configMergeScript} ${nixSettings} "$G_HOME/config.yaml" FIRST=$(cat "$G_HOME/config.yaml") ${configMergeScript} ${nixSettings} "$G_HOME/config.yaml" SECOND=$(cat "$G_HOME/config.yaml") if [ "$FIRST" != "$SECOND" ]; then fail "G: second merge produced different output" echo "--- first ---" echo "$FIRST" echo "--- second ---" echo "$SECOND" fi echo "PASS: Scenario G" # ═══════════════════════════════════════════════════════════════ # Report # ═══════════════════════════════════════════════════════════════ if [ -n "$ERRORS" ]; then echo "" echo "FAILURES:" echo -e "$ERRORS" exit 1 fi echo "" echo "=== All 7 merge scenarios passed ===" mkdir -p $out echo "ok" > $out/result ''; }; }; }