fix: align ollama/fallback model to gemma4:latest (#291) #344

Closed
Timmy wants to merge 2 commits from timmy/issue-291-fix-model-drift into main
9 changed files with 251 additions and 12 deletions

View File

@@ -26,11 +26,11 @@ terminal:
base_url: http://localhost:11434/v1
memory: 5120
container_disk: 51200
container_persistent: true
docker_volumes: []
docker_mount_cwd_to_workspace: false
persistent_shell: true
container_disk: 51200
container_persistent: true
docker_volumes: []
docker_mount_cwd_to_workspace: false
persistent_shell: true
browser:
inactivity_timeout: 120
command_timeout: 30
@@ -181,9 +181,8 @@ mesh:
consensus_mode: competitive
security:
sovereign_audit: true
no_phone_home: true
sovereign_audit: true
no_phone_home: true
redact_secrets: true
tirith_enabled: true
tirith_path: tirith
@@ -225,7 +224,7 @@ DISCORD_HOME_CHANNEL: '1476292315814297772'
providers:
ollama:
base_url: http://localhost:11434/v1
model: hermes3:latest
model: gemma4:latest
mcp_servers:
morrowind:
command: python3
@@ -242,6 +241,6 @@ mcp_servers:
connect_timeout: 60
fallback_model:
provider: ollama
model: hermes3:latest
model: gemma4:latest
base_url: http://localhost:11434/v1
api_key: ''

View File

@@ -60,7 +60,7 @@
"id": "a77a87392582",
"name": "Health Monitor",
"prompt": "Check Ollama is responding, disk space, memory, GPU utilization, process count",
"model": "hermes3:latest",
"model": "gemma4:latest",
"provider": "ollama",
"base_url": "http://localhost:11434/v1",
"schedule": {

View File

@@ -0,0 +1,56 @@
# Poka-Yoke Guards for the Agent Fleet
These guards prevent common failure modes in the Hermes agent fleet.
Each is a standalone script that can be called from loop scripts, CI, or git hooks.
## Guards
### 1. api-key-preflight.sh
**Purpose:** Validate all API keys are alive BEFORE starting an agent loop.
**Usage:** `./api-key-preflight.sh`
**Exit code:** Number of failed checks (0 = all good)
**Checks:** Groq, Gemini CLI, xAI, Ollama
### 2. duplicate-pr-gate.sh
**Purpose:** Prevent duplicate PRs for the same issue.
**Usage:** `./duplicate-pr-gate.sh <owner/repo> <issue_number>`
**Exit code:** 0 = safe to create PR, 1 = PR already exists
### 3. hardcoded-ip-scanner.sh
**Purpose:** Git pre-commit hook that rejects hardcoded VPS IPs.
**Usage:** Symlink into `.git/hooks/pre-commit` or source from existing hook.
**Exit code:** 0 = clean, 1 = hardcoded IPs found
**Blocked IPs:** Hermes (143.198.27.163), Allegro (167.99.126.228), Bezalel (159.203.146.185)
### 4. quality-verify.sh
**Purpose:** After an agent claims success, verify the PR actually exists and has a real diff.
**Usage:** `./quality-verify.sh <owner/repo> <pr_number>`
**Exit code:** 0 = verified real, 1 = fake/empty
### 5. max-attempts.sh
**Purpose:** Track agent attempts per issue. After N failures, skip permanently.
**Usage:** `./max-attempts.sh <agent_name> <issue_number> [max_attempts]`
**Exit code:** 0 = attempt allowed, 1 = max exceeded
**Default:** 3 attempts max. State stored in `~/.hermes/logs/<agent>-attempts.json`
## Integration
```bash
# In a loop script:
source guards/api-key-preflight.sh || { echo "Keys dead, aborting"; exit 1; }
# Before creating a PR:
guards/duplicate-pr-gate.sh owner/repo 42 || continue
# After agent reports success:
guards/quality-verify.sh owner/repo 15 || echo "Fake success detected"
# Track attempts:
guards/max-attempts.sh claude-agent 42 || { echo "Too many failures"; continue; }
```
## Installing as git hook
```bash
ln -sf $(pwd)/hardcoded-ip-scanner.sh /path/to/repo/.git/hooks/pre-commit
```

View File

@@ -0,0 +1,41 @@
#!/bin/bash
# api-key-preflight.sh — validate API keys before burning cycles
# Usage: source this from loop scripts, or run standalone
# Exit code 0 = all keys valid, 1+ = number of dead keys
check_groq() {
KEY=$(cat ~/.config/groq/api_key 2>/dev/null | tr -d '\n')
[ -z "$KEY" ] && echo "GROQ: NO KEY" && return 1
STATUS=$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $KEY" https://api.groq.com/openai/v1/models 2>/dev/null)
[ "$STATUS" = "200" ] && echo "GROQ: OK" && return 0
echo "GROQ: DEAD (HTTP $STATUS)" && return 1
}
check_gemini() {
KEY=$(cat ~/.hermes/gemini_token 2>/dev/null | tr -d '\n')
[ -z "$KEY" ] && echo "GEMINI: NO KEY" && return 1
# Gemini CLI uses its own auth — just check the binary exists
which gemini >/dev/null 2>&1 && echo "GEMINI: CLI OK" && return 0
echo "GEMINI: CLI MISSING" && return 1
}
check_xai() {
KEY=$(grep XAI_API_KEY ~/.hermes/.env 2>/dev/null | cut -d= -f2 | tr -d ' ')
[ -z "$KEY" ] && echo "XAI: NO KEY" && return 1
STATUS=$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $KEY" https://api.x.ai/v1/models 2>/dev/null)
[ "$STATUS" = "200" ] && echo "XAI: OK" && return 0
echo "XAI: DEAD (HTTP $STATUS)" && return 1
}
check_ollama() {
STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:11434/api/version 2>/dev/null)
[ "$STATUS" = "200" ] && echo "OLLAMA: OK" && return 0
echo "OLLAMA: DOWN" && return 1
}
# Run all checks
FAILED=0
for check in check_groq check_gemini check_xai check_ollama; do
$check || FAILED=$((FAILED+1))
done
exit $FAILED

View File

@@ -0,0 +1,41 @@
#!/bin/bash
# duplicate-pr-gate.sh <repo> <issue_number>
# Exit 0 = no existing PR, safe to create
# Exit 1 = PR already exists, skip
REPO="$1"
ISSUE="$2"
TOKEN=$(cat ~/.hermes/gitea_token_vps 2>/dev/null | tr -d '\n')
API="https://forge.alexanderwhitestone.com/api/v1"
if [ -z "$REPO" ] || [ -z "$ISSUE" ]; then
echo "Usage: duplicate-pr-gate.sh <owner/repo> <issue_number>"
exit 2
fi
if [ -z "$TOKEN" ]; then
echo "ERROR: No Gitea token found at ~/.hermes/gitea_token_vps"
exit 2
fi
# Check for open PRs mentioning this issue
EXISTING=$(curl -s "$API/repos/$REPO/pulls?state=open&limit=50" \
-H "Authorization: token $TOKEN" | \
python3 -c "
import sys,json
try:
prs=json.load(sys.stdin)
if not isinstance(prs, list):
print(0)
sys.exit()
matches=[p['number'] for p in prs if '#$ISSUE' in p.get('title','') or '#$ISSUE' in p.get('body','') or 'issue-$ISSUE' in p.get('head',{}).get('ref','')]
print(len(matches))
except:
print(0)
" 2>/dev/null)
if [ "$EXISTING" -gt 0 ] 2>/dev/null; then
echo "BLOCKED: PR already exists for issue #$ISSUE on $REPO"
exit 1
fi
echo "CLEAR: No existing PR for #$ISSUE"
exit 0

View File

@@ -0,0 +1,22 @@
#!/bin/bash
# hardcoded-ip-scanner.sh — scan staged files for hardcoded VPS IPs
# Add to any repo's pre-commit hook
# Our VPS IPs that should NEVER be hardcoded (use domains instead)
BAD_PATTERNS=(
'143.198.27.163:3000' # Hermes VPS Gitea — use forge.alexanderwhitestone.com
'167.99.126.228:3000' # Allegro VPS — use allegro.alexanderwhitestone.com
'159.203.146.185:3000' # Bezalel VPS — use bezalel.alexanderwhitestone.com
)
FOUND=0
for pattern in "${BAD_PATTERNS[@]}"; do
MATCHES=$(git diff --cached --name-only | xargs grep -l "$pattern" 2>/dev/null)
if [ -n "$MATCHES" ]; then
echo "BLOCKED: Hardcoded IP '$pattern' found in: $MATCHES"
echo "Use the domain name instead."
FOUND=1
fi
done
[ $FOUND -eq 1 ] && exit 1
exit 0

View File

@@ -0,0 +1,37 @@
#!/bin/bash
# max-attempts.sh <agent> <issue_number> [max_attempts]
# Exit 0 = attempt allowed, Exit 1 = max attempts exceeded, skip permanently
AGENT="$1"
ISSUE="$2"
MAX="${3:-3}"
ATTEMPTS_FILE="$HOME/.hermes/logs/${AGENT}-attempts.json"
if [ -z "$AGENT" ] || [ -z "$ISSUE" ]; then
echo "Usage: max-attempts.sh <agent_name> <issue_number> [max_attempts]"
exit 2
fi
# Ensure logs dir exists
mkdir -p "$HOME/.hermes/logs"
# Initialize if needed
[ ! -f "$ATTEMPTS_FILE" ] && echo '{}' > "$ATTEMPTS_FILE"
# Read current count
COUNT=$(python3 -c "import json; d=json.load(open('$ATTEMPTS_FILE')); print(d.get('$ISSUE',0))" 2>/dev/null)
COUNT=${COUNT:-0}
if [ "$COUNT" -ge "$MAX" ]; then
echo "SKIP: Issue #$ISSUE exceeded $MAX attempts by $AGENT"
exit 1
fi
# Increment
python3 -c "
import json
with open('$ATTEMPTS_FILE') as f: d=json.load(f)
d['$ISSUE'] = d.get('$ISSUE',0) + 1
with open('$ATTEMPTS_FILE','w') as f: json.dump(d,f,indent=2)
print(f'ATTEMPT {d[\"$ISSUE\"]}/$MAX for #$ISSUE by $AGENT')
"
exit 0

View File

@@ -0,0 +1,43 @@
#!/bin/bash
# quality-verify.sh <repo> <pr_number>
# Exit 0 = quality verified, Exit 1 = fake/empty success
REPO="$1"
PR="$2"
TOKEN=$(cat ~/.hermes/gitea_token_vps 2>/dev/null | tr -d '\n')
API="https://forge.alexanderwhitestone.com/api/v1"
if [ -z "$REPO" ] || [ -z "$PR" ]; then
echo "Usage: quality-verify.sh <owner/repo> <pr_number>"
exit 2
fi
if [ -z "$TOKEN" ]; then
echo "ERROR: No Gitea token found at ~/.hermes/gitea_token_vps"
exit 2
fi
# Check PR exists
DATA=$(curl -s "$API/repos/$REPO/pulls/$PR" -H "Authorization: token $TOKEN")
STATE=$(echo "$DATA" | python3 -c "import sys,json; print(json.load(sys.stdin).get('state','missing'))" 2>/dev/null)
if [ "$STATE" = "missing" ] || [ -z "$STATE" ]; then
echo "FAIL: PR #$PR does not exist on $REPO"
exit 1
fi
# Check it has actual file changes
FILES=$(curl -s "$API/repos/$REPO/pulls/$PR/files" -H "Authorization: token $TOKEN" | \
python3 -c "import sys,json; print(len(json.load(sys.stdin)))" 2>/dev/null)
if [ "$FILES" = "0" ] || [ -z "$FILES" ]; then
echo "FAIL: PR #$PR has zero file changes"
exit 1
fi
# Check it's mergeable
MERGEABLE=$(echo "$DATA" | python3 -c "import sys,json; print(json.load(sys.stdin).get('mergeable',False))" 2>/dev/null)
if [ "$MERGEABLE" != "True" ]; then
echo "WARN: PR #$PR may not be mergeable (conflicts possible)"
# Don't fail — just warn. Conflicts can be fixed.
fi
echo "VERIFIED: PR #$PR on $REPO$FILES files changed, state=$STATE, mergeable=$MERGEABLE"
exit 0

View File

@@ -18,5 +18,5 @@ def test_config_defaults_to_local_llama_cpp_runtime() -> None:
assert local_provider["model"] == "hermes4:14b"
assert config["fallback_model"]["provider"] == "ollama"
assert config["fallback_model"]["model"] == "hermes3:latest"
assert config["fallback_model"]["model"] == "gemma4:latest"
assert "localhost" in config["fallback_model"]["base_url"]