bin/claude-loop.sh

@@ -11,7 +11,7 @@ set -euo pipefail
 NUM_WORKERS="${1:-2}"
 MAX_WORKERS=10        # absolute ceiling
 WORKTREE_BASE="$HOME/worktrees"
-GITEA_URL="http://143.198.27.163:3000"
+GITEA_URL="${GITEA_URL:-https://forge.alexanderwhitestone.com}"
 GITEA_TOKEN=$(cat "$HOME/.hermes/claude_token")
 CLAUDE_TIMEOUT=900   # 15 min per issue
 COOLDOWN=15           # seconds between issues — stagger clones

bin/gemini-loop.sh

@@ -7,13 +7,26 @@
 
 set -euo pipefail
 
-export GEMINI_API_KEY="AIzaSyAmGgS516K4PwlODFEnghL535yzoLnofKM"
+GEMINI_KEY_FILE="${GEMINI_KEY_FILE:-$HOME/.timmy/gemini_free_tier_key}"
+if [ -f "$GEMINI_KEY_FILE" ]; then
+  export GEMINI_API_KEY="$(python3 - "$GEMINI_KEY_FILE" <<'PY'
+from pathlib import Path
+import sys
+text = Path(sys.argv[1]).read_text(errors='ignore').splitlines()
+for line in text:
+    line=line.strip()
+    if line:
+        print(line)
+        break
+PY
+)"
+fi
 
 # === CONFIG ===
 NUM_WORKERS="${1:-2}"
 MAX_WORKERS=5
 WORKTREE_BASE="$HOME/worktrees"
-GITEA_URL="http://143.198.27.163:3000"
+GITEA_URL="${GITEA_URL:-https://forge.alexanderwhitestone.com}"
 GITEA_TOKEN=$(cat "$HOME/.hermes/gemini_token")
 GEMINI_TIMEOUT=600   # 10 min per issue
 COOLDOWN=15           # seconds between issues — stagger clones
@@ -24,6 +37,7 @@ SKIP_FILE="$LOG_DIR/gemini-skip-list.json"
 LOCK_DIR="$LOG_DIR/gemini-locks"
 ACTIVE_FILE="$LOG_DIR/gemini-active.json"
 ALLOW_SELF_ASSIGN="${ALLOW_SELF_ASSIGN:-0}"   # 0 = only explicitly-assigned Gemini work
+AUTH_INVALID_SLEEP=900
 
 mkdir -p "$LOG_DIR" "$WORKTREE_BASE" "$LOCK_DIR"
 [ -f "$SKIP_FILE" ] || echo '{}' > "$SKIP_FILE"
@@ -34,6 +48,124 @@ log() {
   echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_DIR/gemini-loop.log"
 }
 
+post_issue_comment() {
+  local repo_owner="$1" repo_name="$2" issue_num="$3" body="$4"
+  local payload
+  payload=$(python3 - "$body" <<'PY'
+import json, sys
+print(json.dumps({"body": sys.argv[1]}))
+PY
+)
+  curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}/comments"     -H "Authorization: token ${GITEA_TOKEN}"     -H "Content-Type: application/json"     -d "$payload" >/dev/null 2>&1 || true
+}
+
+remote_branch_exists() {
+  local branch="$1"
+  git ls-remote --heads origin "$branch" 2>/dev/null | grep -q .
+}
+
+get_pr_num() {
+  local repo_owner="$1" repo_name="$2" branch="$3"
+  curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls?state=all&head=${repo_owner}:${branch}&limit=1"     -H "Authorization: token ${GITEA_TOKEN}" | python3 -c "
+import sys,json
+prs = json.load(sys.stdin)
+if prs: print(prs[0]['number'])
+else: print('')
+" 2>/dev/null
+}
+
+get_pr_file_count() {
+  local repo_owner="$1" repo_name="$2" pr_num="$3"
+  curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/files"     -H "Authorization: token ${GITEA_TOKEN}" | python3 -c "
+import sys, json
+try:
+    files = json.load(sys.stdin)
+    print(len(files) if isinstance(files, list) else 0)
+except:
+    print(0)
+" 2>/dev/null
+}
+
+get_pr_state() {
+  local repo_owner="$1" repo_name="$2" pr_num="$3"
+  curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}"     -H "Authorization: token ${GITEA_TOKEN}" | python3 -c "
+import sys, json
+try:
+    pr = json.load(sys.stdin)
+    if pr.get('merged'):
+        print('merged')
+    else:
+        print(pr.get('state', 'unknown'))
+except:
+    print('unknown')
+" 2>/dev/null
+}
+
+get_issue_state() {
+  local repo_owner="$1" repo_name="$2" issue_num="$3"
+  curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}"     -H "Authorization: token ${GITEA_TOKEN}" | python3 -c "
+import sys, json
+try:
+    issue = json.load(sys.stdin)
+    print(issue.get('state', 'unknown'))
+except:
+    print('unknown')
+" 2>/dev/null
+}
+
+proof_comment_status() {
+  local repo_owner="$1" repo_name="$2" issue_num="$3" branch="$4"
+  curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}/comments"     -H "Authorization: token ${GITEA_TOKEN}" | BRANCH="$branch" python3 -c "
+import os, sys, json
+branch = os.environ.get('BRANCH', '').lower()
+try:
+    comments = json.load(sys.stdin)
+except Exception:
+    print('missing|')
+    raise SystemExit(0)
+for c in reversed(comments):
+    user = ((c.get('user') or {}).get('login') or '').lower()
+    body = c.get('body') or ''
+    body_l = body.lower()
+    if user != 'gemini':
+        continue
+    if 'proof:' not in body_l and 'verification:' not in body_l:
+        continue
+    has_branch = branch in body_l
+    has_pr = ('pr:' in body_l) or ('pull request:' in body_l) or ('/pulls/' in body_l)
+    has_push = ('push:' in body_l) or ('pushed' in body_l)
+    has_verify = ('tox' in body_l) or ('pytest' in body_l) or ('verification:' in body_l) or ('npm test' in body_l)
+    status = 'ok' if (has_branch and has_pr and has_push and has_verify) else 'incomplete'
+    print(status + '|' + (c.get('html_url') or ''))
+    raise SystemExit(0)
+print('missing|')
+" 2>/dev/null
+}
+
+gemini_auth_invalid() {
+  local issue_num="$1"
+  grep -q "API_KEY_INVALID\|API key expired" "$LOG_DIR/gemini-${issue_num}.log" 2>/dev/null
+}
+
+issue_is_code_fit() {
+  local title="$1"
+  local labels="$2"
+  local body="$3"
+  local haystack
+  haystack="${title} ${labels} ${body}"
+  local low="${haystack,,}"
+
+  if [[ "$low" == *"[morning report]"* ]]; then return 1; fi
+  if [[ "$low" == *"[kt]"* ]]; then return 1; fi
+  if [[ "$low" == *"policy:"* ]]; then return 1; fi
+  if [[ "$low" == *"incident:"* || "$low" == *"🚨 incident"* || "$low" == *"[incident]"* ]]; then return 1; fi
+  if [[ "$low" == *"fleet lexicon"* || "$low" == *"shared vocabulary"* || "$low" == *"rubric"* ]]; then return 1; fi
+  if [[ "$low" == *"archive ghost"* || "$low" == *"reassign"* || "$low" == *"offload"* || "$low" == *"burn directive"* ]]; then return 1; fi
+  if [[ "$low" == *"review all open prs"* ]]; then return 1; fi
+  if [[ "$low" == *"epic"* ]]; then return 1; fi
+  return 0
+}
+
 lock_issue() {
   local issue_key="$1"
   local lockfile="$LOCK_DIR/$issue_key.lock"
@@ -90,6 +222,7 @@ with open('$ACTIVE_FILE', 'r+') as f:
 
 cleanup_workdir() {
   local wt="$1"
+  cd "$HOME" 2>/dev/null || true
   rm -rf "$wt" 2>/dev/null || true
 }
 
@@ -154,8 +287,11 @@ for i in all_issues:
         continue
 
     title = i['title'].lower()
+    labels = [l['name'].lower() for l in (i.get('labels') or [])]
+    body = (i.get('body') or '').lower()
     if '[philosophy]' in title: continue
     if '[epic]' in title or 'epic:' in title: continue
+    if 'epic' in labels: continue
     if '[showcase]' in title: continue
     if '[do not close' in title: continue
     if '[meta]' in title: continue
@@ -164,6 +300,11 @@ for i in all_issues:
     if '[morning report]' in title: continue
     if '[retro]' in title: continue
     if '[intel]' in title: continue
+    if '[kt]' in title: continue
+    if 'policy:' in title: continue
+    if 'incident' in title: continue
+    if 'lexicon' in title or 'shared vocabulary' in title or 'rubric' in title: continue
+    if 'archive ghost' in title or 'reassign' in title or 'offload' in title: continue
     if 'master escalation' in title: continue
     if any(a['login'] == 'Rockachopa' for a in (i.get('assignees') or [])): continue
 
@@ -250,10 +391,11 @@ You can do ANYTHING a developer can do.
 - If tests fail after 2 attempts, STOP and comment on the issue explaining why.
 - Be thorough but focused. Fix the issue, don't refactor the world.
 
-== CRITICAL: ALWAYS COMMIT AND PUSH ==
+== CRITICAL: FINISH = PUSHED + PR'D + PROVED ==
 - NEVER exit without committing your work. Even partial progress MUST be committed.
 - Before you finish, ALWAYS: git add -A && git commit && git push origin gemini/issue-${issue_num}
 - ALWAYS create a PR before exiting. No exceptions.
+- ALWAYS post the Proof block before exiting. No proof comment = not done.
 - If a branch already exists with prior work, check it out and CONTINUE from where it left off.
 - Check: git ls-remote origin gemini/issue-${issue_num} — if it exists, pull it first.
 - Your work is WASTED if it's not pushed. Push early, push often.
@@ -364,19 +506,10 @@ Work in progress, may need continuation." 2>/dev/null || true
     fi
 
     # ── Create PR if needed ──
-    pr_num=$(curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls?state=open&head=${repo_owner}:${branch}&limit=1" \
-      -H "Authorization: token ${GITEA_TOKEN}" | python3 -c "
-import sys,json
-prs = json.load(sys.stdin)
-if prs: print(prs[0]['number'])
-else: print('')
-" 2>/dev/null)
+    pr_num=$(get_pr_num "$repo_owner" "$repo_name" "$branch")
 
     if [ -z "$pr_num" ] && [ "${UNPUSHED:-0}" -gt 0 ]; then
-      pr_num=$(curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls" \
-        -H "Authorization: token ${GITEA_TOKEN}" \
-        -H "Content-Type: application/json" \
-        -d "$(python3 -c "
+      pr_num=$(curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls"         -H "Authorization: token ${GITEA_TOKEN}"         -H "Content-Type: application/json"         -d "$(python3 -c "
 import json
 print(json.dumps({
     'title': 'Gemini: Issue #${issue_num}',
@@ -388,26 +521,72 @@ print(json.dumps({
       [ -n "$pr_num" ] && log "WORKER-${worker_id}: Created PR #${pr_num} for issue #${issue_num}"
     fi
 
-    # ── Merge + close on success ──
+    # ── Verify finish semantics / classify failures ──
     if [ "$exit_code" -eq 0 ]; then
-      log "WORKER-${worker_id}: SUCCESS #${issue_num}"
-      if [ -n "$pr_num" ]; then
-        curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/merge" \
-          -H "Authorization: token ${GITEA_TOKEN}" \
-          -H "Content-Type: application/json" \
-          -d '{"Do": "squash"}' >/dev/null 2>&1 || true
-        curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}" \
-          -H "Authorization: token ${GITEA_TOKEN}" \
-          -H "Content-Type: application/json" \
-          -d '{"state": "closed"}' >/dev/null 2>&1 || true
-        log "WORKER-${worker_id}: PR #${pr_num} merged, issue #${issue_num} closed"
+      log "WORKER-${worker_id}: SUCCESS #${issue_num} exited 0 — verifying push + PR + proof"
+      if ! remote_branch_exists "$branch"; then
+        log "WORKER-${worker_id}: BLOCKED #${issue_num} remote branch missing"
+        post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "Loop gate blocked completion: remote branch ${branch} was not found on origin after Gemini exited. Issue remains open for retry."
+        mark_skip "$issue_num" "missing_remote_branch" 1
+        consecutive_failures=$((consecutive_failures + 1))
+      elif [ -z "$pr_num" ]; then
+        log "WORKER-${worker_id}: BLOCKED #${issue_num} no PR found"
+        post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "Loop gate blocked completion: branch ${branch} exists remotely, but no PR was found. Issue remains open for retry."
+        mark_skip "$issue_num" "missing_pr" 1
+        consecutive_failures=$((consecutive_failures + 1))
+      else
+        pr_files=$(get_pr_file_count "$repo_owner" "$repo_name" "$pr_num")
+        if [ "${pr_files:-0}" -eq 0 ]; then
+          log "WORKER-${worker_id}: BLOCKED #${issue_num} PR #${pr_num} has 0 changed files"
+          curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}"             -H "Authorization: token ${GITEA_TOKEN}"             -H "Content-Type: application/json"             -d '{"state": "closed"}' >/dev/null 2>&1 || true
+          post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "PR #${pr_num} was closed automatically: it had 0 changed files (empty commit). Issue remains open for retry."
+          mark_skip "$issue_num" "empty_commit" 2
+          consecutive_failures=$((consecutive_failures + 1))
+        else
+          proof_status=$(proof_comment_status "$repo_owner" "$repo_name" "$issue_num" "$branch")
+          proof_state="${proof_status%%|*}"
+          proof_url="${proof_status#*|}"
+          if [ "$proof_state" != "ok" ]; then
+            log "WORKER-${worker_id}: BLOCKED #${issue_num} proof missing or incomplete (${proof_state})"
+            post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "Loop gate blocked completion: PR #${pr_num} exists and has ${pr_files} changed file(s), but the required Proof block from Gemini is missing or incomplete. Issue remains open for retry."
+            mark_skip "$issue_num" "missing_proof" 1
+            consecutive_failures=$((consecutive_failures + 1))
+          else
+            log "WORKER-${worker_id}: PROOF verified ${proof_url}"
+            pr_state=$(get_pr_state "$repo_owner" "$repo_name" "$pr_num")
+            if [ "$pr_state" = "open" ]; then
+              curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/merge"                 -H "Authorization: token ${GITEA_TOKEN}"                 -H "Content-Type: application/json"                 -d '{"Do": "squash"}' >/dev/null 2>&1 || true
+              pr_state=$(get_pr_state "$repo_owner" "$repo_name" "$pr_num")
+            fi
+            if [ "$pr_state" = "merged" ]; then
+              curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}"                 -H "Authorization: token ${GITEA_TOKEN}"                 -H "Content-Type: application/json"                 -d '{"state": "closed"}' >/dev/null 2>&1 || true
+              issue_state=$(get_issue_state "$repo_owner" "$repo_name" "$issue_num")
+              if [ "$issue_state" = "closed" ]; then
+                log "WORKER-${worker_id}: VERIFIED #${issue_num} branch pushed, PR merged, proof present, issue closed"
+                consecutive_failures=0
+              else
+                log "WORKER-${worker_id}: BLOCKED #${issue_num} issue did not close after merge"
+                mark_skip "$issue_num" "issue_close_unverified" 1
+                consecutive_failures=$((consecutive_failures + 1))
+              fi
+            else
+              log "WORKER-${worker_id}: BLOCKED #${issue_num} merge not verified (state=${pr_state})"
+              mark_skip "$issue_num" "merge_unverified" 1
+              consecutive_failures=$((consecutive_failures + 1))
+            fi
+          fi
+        fi
       fi
-      consecutive_failures=0
     elif [ "$exit_code" -eq 124 ]; then
       log "WORKER-${worker_id}: TIMEOUT #${issue_num} (work saved in PR)"
       consecutive_failures=$((consecutive_failures + 1))
     else
-      if grep -q "rate_limit\|rate limit\|429\|overloaded\|quota" "$LOG_DIR/gemini-${issue_num}.log" 2>/dev/null; then
+      if gemini_auth_invalid "$issue_num"; then
+        log "WORKER-${worker_id}: AUTH INVALID on #${issue_num} — sleeping ${AUTH_INVALID_SLEEP}s"
+        mark_skip "$issue_num" "gemini_auth_invalid" 1
+        sleep "$AUTH_INVALID_SLEEP"
+        consecutive_failures=$((consecutive_failures + 5))
+      elif grep -q "rate_limit\|rate limit\|429\|overloaded\|quota" "$LOG_DIR/gemini-${issue_num}.log" 2>/dev/null; then
         log "WORKER-${worker_id}: RATE LIMITED on #${issue_num} (work saved)"
         consecutive_failures=$((consecutive_failures + 3))
       else
@@ -444,7 +623,7 @@ print(json.dumps({
     'pr': '${pr_num:-}',
     'merged': $( [ '$OUTCOME' = 'success' ] && [ -n '${pr_num:-}' ] && echo 'true' || echo 'false' )
 }))
-" >> "$LOG_DIR/claude-metrics.jsonl" 2>/dev/null
+" >> "$LOG_DIR/gemini-metrics.jsonl" 2>/dev/null
 
     cleanup_workdir "$worktree"
     unlock_issue "$issue_key"

bin/timmy-orchestrator.sh

@@ -8,7 +8,7 @@ set -uo pipefail
 LOG_DIR="$HOME/.hermes/logs"
 LOG="$LOG_DIR/timmy-orchestrator.log"
 PIDFILE="$LOG_DIR/timmy-orchestrator.pid"
-GITEA_URL="http://143.198.27.163:3000"
+GITEA_URL="${GITEA_URL:-https://forge.alexanderwhitestone.com}"
 GITEA_TOKEN=$(cat "$HOME/.hermes/gitea_token_vps" 2>/dev/null)  # Timmy token, NOT rockachopa
 CYCLE_INTERVAL=300
 HERMES_TIMEOUT=180
@@ -66,10 +66,10 @@ for p in json.load(sys.stdin):
   echo "Claude loop: $(pgrep -f 'claude-loop.sh' 2>/dev/null | wc -l | tr -d ' ') procs" >> "$state_dir/agent_status.txt"
   tail -50 "$LOG_DIR/claude-loop.log" 2>/dev/null | grep -c "SUCCESS" | xargs -I{} echo "Claude recent successes: {}" >> "$state_dir/agent_status.txt"
   tail -50 "$LOG_DIR/claude-loop.log" 2>/dev/null | grep -c "FAILED" | xargs -I{} echo "Claude recent failures: {}" >> "$state_dir/agent_status.txt"
-  echo "Kimi loop: $(pgrep -f 'kimi-loop.sh' 2>/dev/null | wc -l | tr -d ' ') procs" >> "$state_dir/agent_status.txt"
-  tail -50 "$LOG_DIR/kimi-loop.log" 2>/dev/null | grep -c "SUCCESS" | xargs -I{} echo "Kimi recent successes: {}" >> "$state_dir/agent_status.txt"
-  tail -50 "$LOG_DIR/kimi-loop.log" 2>/dev/null | grep -c "FAILED" | xargs -I{} echo "Kimi recent failures: {}" >> "$state_dir/agent_status.txt"
-  tail -1 "$LOG_DIR/kimi-loop.log" 2>/dev/null | xargs -I{} echo "Kimi last event: {}" >> "$state_dir/agent_status.txt"
+  echo "Kimi heartbeat launchd: $(launchctl list 2>/dev/null | grep -c 'ai.timmy.kimi-heartbeat' | tr -d ' ') job" >> "$state_dir/agent_status.txt"
+  tail -50 "/tmp/kimi-heartbeat.log" 2>/dev/null | grep -c "DISPATCHED:" | xargs -I{} echo "Kimi recent dispatches: {}" >> "$state_dir/agent_status.txt"
+  tail -50 "/tmp/kimi-heartbeat.log" 2>/dev/null | grep -c "FAILED:" | xargs -I{} echo "Kimi recent failures: {}" >> "$state_dir/agent_status.txt"
+  tail -1 "/tmp/kimi-heartbeat.log" 2>/dev/null | xargs -I{} echo "Kimi last event: {}" >> "$state_dir/agent_status.txt"
 
   echo "$state_dir"
 }