From e5055d269bf7b0e71cbe9e55ef1ffb43db0ed5ea Mon Sep 17 00:00:00 2001 From: ezra Date: Tue, 7 Apr 2026 16:12:05 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20genchi-genbutsu=20=E2=80=94=20verify=20?= =?UTF-8?q?world=20state,=20not=20log=20vibes=20(#348)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement 現地現物 (Genchi Genbutsu) post-completion verification: - Add bin/genchi-genbutsu.sh performing 5 world-state checks: 1. Branch exists on remote 2. PR exists 3. PR has real file changes (> 0) 4. PR is mergeable 5. Issue has a completion comment from the agent - Wire verification into all agent loops: - bin/claude-loop.sh: call genchi-genbutsu before merge/close - bin/gemini-loop.sh: delegate existing inline checks to genchi-genbutsu - bin/agent-loop.sh: resurrect generic agent loop with genchi-genbutsu wired in - Update metrics JSONL to include 'verified' field for all loops - Update burn monitor (tasks.py velocity_tracking): - Report verified_completion count alongside raw completions - Dashboard shows verified trend history - Update morning report (tasks.py good_morning_report): - Count only verified completions from the last 24h - Surface verification failures in the report body Fixes #348 Refs #345 --- bin/agent-loop.sh | 273 +++++++++++++++++++++++++++++++++++++++++ bin/claude-loop.sh | 42 ++++--- bin/gemini-loop.sh | 103 ++++++++-------- bin/genchi-genbutsu.sh | 179 +++++++++++++++++++++++++++ tasks.py | 89 ++++++++++++-- 5 files changed, 609 insertions(+), 77 deletions(-) create mode 100755 bin/agent-loop.sh create mode 100755 bin/genchi-genbutsu.sh diff --git a/bin/agent-loop.sh b/bin/agent-loop.sh new file mode 100755 index 00000000..d2e56bc0 --- /dev/null +++ b/bin/agent-loop.sh @@ -0,0 +1,273 @@ +#!/usr/bin/env bash +# agent-loop.sh — Universal agent dev loop with Genchi Genbutsu verification +# +# Usage: agent-loop.sh [num-workers] +# agent-loop.sh claude 2 +# agent-loop.sh gemini 1 +# +# Dispatches via agent-dispatch.sh, then verifies with genchi-genbutsu.sh. + +set -uo pipefail + +AGENT="${1:?Usage: agent-loop.sh [num-workers]}" +NUM_WORKERS="${2:-1}" + +# Resolve agent tool and model from config or fallback +case "$AGENT" in + claude) TOOL="claude"; MODEL="sonnet" ;; + gemini) TOOL="gemini"; MODEL="gemini-2.5-pro-preview-05-06" ;; + grok) TOOL="opencode"; MODEL="grok-3-fast" ;; + *) TOOL="$AGENT"; MODEL="" ;; +esac + +# === CONFIG === +GITEA_URL="${GITEA_URL:-https://forge.alexanderwhitestone.com}" +GITEA_TOKEN="${GITEA_TOKEN:-}" +WORKTREE_BASE="$HOME/worktrees" +LOG_DIR="$HOME/.hermes/logs" +LOCK_DIR="$LOG_DIR/${AGENT}-locks" +SKIP_FILE="$LOG_DIR/${AGENT}-skip-list.json" +ACTIVE_FILE="$LOG_DIR/${AGENT}-active.json" +TIMEOUT=600 +COOLDOWN=30 + +mkdir -p "$LOG_DIR" "$WORKTREE_BASE" "$LOCK_DIR" +[ -f "$SKIP_FILE" ] || echo '{}' > "$SKIP_FILE" +echo '{}' > "$ACTIVE_FILE" + +# === SHARED FUNCTIONS === +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ${AGENT}: $*" >> "$LOG_DIR/${AGENT}-loop.log" +} + +lock_issue() { + local key="$1" + mkdir "$LOCK_DIR/$key.lock" 2>/dev/null && echo $$ > "$LOCK_DIR/$key.lock/pid" +} + +unlock_issue() { + rm -rf "$LOCK_DIR/$1.lock" 2>/dev/null +} + +mark_skip() { + local issue_num="$1" reason="$2" + python3 -c " +import json, time, fcntl +with open('${SKIP_FILE}', 'r+') as f: + fcntl.flock(f, fcntl.LOCK_EX) + try: skips = json.load(f) + except: skips = {} + failures = skips.get(str($issue_num), {}).get('failures', 0) + 1 + skip_hours = 6 if failures >= 3 else 1 + skips[str($issue_num)] = {'until': time.time() + (skip_hours * 3600), 'reason': '$reason', 'failures': failures} + f.seek(0); f.truncate() + json.dump(skips, f, indent=2) +" 2>/dev/null +} + +get_next_issue() { + python3 -c " +import json, sys, time, urllib.request, os +token = '${GITEA_TOKEN}' +base = '${GITEA_URL}' +repos = ['Timmy_Foundation/the-nexus', 'Timmy_Foundation/timmy-config', 'Timmy_Foundation/hermes-agent'] +try: + with open('${SKIP_FILE}') as f: skips = json.load(f) +except: skips = {} +try: + with open('${ACTIVE_FILE}') as f: active = json.load(f); active_issues = {v['issue'] for v in active.values()} +except: active_issues = set() +all_issues = [] +for repo in repos: + url = f'{base}/api/v1/repos/{repo}/issues?state=open&type=issues&limit=50&sort=created' + req = urllib.request.Request(url, headers={'Authorization': f'token {token}'}) + try: + resp = urllib.request.urlopen(req, timeout=10) + issues = json.loads(resp.read()) + for i in issues: i['_repo'] = repo + all_issues.extend(issues) + except: continue +for i in sorted(all_issues, key=lambda x: x['title'].lower()): + assignees = [a['login'] for a in (i.get('assignees') or [])] + if assignees and '${AGENT}' not in assignees: continue + num_str = str(i['number']) + if num_str in active_issues: continue + if skips.get(num_str, {}).get('until', 0) > time.time(): continue + lock = '${LOCK_DIR}/' + i['_repo'].replace('/', '-') + '-' + num_str + '.lock' + if os.path.isdir(lock): continue + owner, name = i['_repo'].split('/') + print(json.dumps({'number': i['number'], 'title': i['title'], 'repo_owner': owner, 'repo_name': name, 'repo': i['_repo']})) + sys.exit(0) +print('null') +" 2>/dev/null +} + +# === WORKER FUNCTION === +run_worker() { + local worker_id="$1" + log "WORKER-${worker_id}: Started" + + while true; do + issue_json=$(get_next_issue) + if [ "$issue_json" = "null" ] || [ -z "$issue_json" ]; then + sleep 30 + continue + fi + + issue_num=$(echo "$issue_json" | python3 -c "import sys,json; print(json.load(sys.stdin)['number'])") + issue_title=$(echo "$issue_json" | python3 -c "import sys,json; print(json.load(sys.stdin)['title'])") + repo_owner=$(echo "$issue_json" | python3 -c "import sys,json; print(json.load(sys.stdin)['repo_owner'])") + repo_name=$(echo "$issue_json" | python3 -c "import sys,json; print(json.load(sys.stdin)['repo_name'])") + issue_key="${repo_owner}-${repo_name}-${issue_num}" + branch="${AGENT}/issue-${issue_num}" + worktree="${WORKTREE_BASE}/${AGENT}-w${worker_id}-${issue_num}" + + if ! lock_issue "$issue_key"; then + sleep 5 + continue + fi + + log "WORKER-${worker_id}: === ISSUE #${issue_num}: ${issue_title} (${repo_owner}/${repo_name}) ===" + + # Clone / checkout + rm -rf "$worktree" 2>/dev/null + CLONE_URL="http://${AGENT}:${GITEA_TOKEN}@143.198.27.163:3000/${repo_owner}/${repo_name}.git" + if git ls-remote --heads "$CLONE_URL" "$branch" 2>/dev/null | grep -q "$branch"; then + git clone --depth=50 -b "$branch" "$CLONE_URL" "$worktree" >/dev/null 2>&1 + else + git clone --depth=1 -b main "$CLONE_URL" "$worktree" >/dev/null 2>&1 + cd "$worktree" && git checkout -b "$branch" >/dev/null 2>&1 + fi + cd "$worktree" + + # Generate prompt + prompt=$(bash "$(dirname "$0")/agent-dispatch.sh" "$AGENT" "$issue_num" "${repo_owner}/${repo_name}") + + CYCLE_START=$(date +%s) + set +e + if [ "$TOOL" = "claude" ]; then + env -u CLAUDECODE gtimeout "$TIMEOUT" claude \ + --print --model "$MODEL" --dangerously-skip-permissions \ + -p "$prompt" > "$LOG_DIR/${AGENT}-${issue_num}.log" 2>&1 + elif [ "$TOOL" = "gemini" ]; then + gtimeout "$TIMEOUT" gemini -p "$prompt" --yolo \ + > "$LOG_DIR/${AGENT}-${issue_num}.log" 2>&1 + else + gtimeout "$TIMEOUT" "$TOOL" "$prompt" \ + > "$LOG_DIR/${AGENT}-${issue_num}.log" 2>&1 + fi + exit_code=$? + set -e + CYCLE_END=$(date +%s) + CYCLE_DURATION=$((CYCLE_END - CYCLE_START)) + + # Salvage + cd "$worktree" 2>/dev/null || true + DIRTY=$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ') + if [ "${DIRTY:-0}" -gt 0 ]; then + git add -A 2>/dev/null + git commit -m "WIP: ${AGENT} progress on #${issue_num} + +Automated salvage commit — agent session ended (exit $exit_code)." 2>/dev/null || true + fi + + UNPUSHED=$(git log --oneline "origin/main..HEAD" 2>/dev/null | wc -l | tr -d ' ') + if [ "${UNPUSHED:-0}" -gt 0 ]; then + git push -u origin "$branch" 2>/dev/null && \ + log "WORKER-${worker_id}: Pushed $UNPUSHED commit(s) on $branch" || \ + log "WORKER-${worker_id}: Push failed for $branch" + fi + + # Create PR if needed + pr_num=$(curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls?state=open&head=${repo_owner}:${branch}&limit=1" \ + -H "Authorization: token ${GITEA_TOKEN}" | python3 -c " +import sys,json +prs = json.load(sys.stdin) +print(prs[0]['number'] if prs else '') +" 2>/dev/null) + + if [ -z "$pr_num" ] && [ "${UNPUSHED:-0}" -gt 0 ]; then + pr_num=$(curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$(python3 -c " +import json +print(json.dumps({ + 'title': '${AGENT}: Issue #${issue_num}', + 'head': '${branch}', + 'base': 'main', + 'body': 'Automated PR for issue #${issue_num}.\nExit code: ${exit_code}' +})) +")" | python3 -c "import sys,json; print(json.load(sys.stdin).get('number',''))" 2>/dev/null) + [ -n "$pr_num" ] && log "WORKER-${worker_id}: Created PR #${pr_num} for issue #${issue_num}" + fi + + # ── Genchi Genbutsu: verify world state before declaring success ── + VERIFIED="false" + if [ "$exit_code" -eq 0 ]; then + log "WORKER-${worker_id}: SUCCESS #${issue_num} — running genchi-genbutsu" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + if verify_result=$("$SCRIPT_DIR/genchi-genbutsu.sh" "$repo_owner" "$repo_name" "$issue_num" "$branch" "$AGENT" 2>/dev/null); then + VERIFIED="true" + log "WORKER-${worker_id}: VERIFIED #${issue_num}" + if [ -n "$pr_num" ]; then + curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/merge" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"Do": "squash"}' >/dev/null 2>&1 || true + curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"state": "closed"}' >/dev/null 2>&1 || true + log "WORKER-${worker_id}: PR #${pr_num} merged, issue #${issue_num} closed" + fi + consecutive_failures=0 + else + verify_details=$(echo "$verify_result" | python3 -c "import sys,json; print(json.load(sys.stdin).get('details','unknown'))" 2>/dev/null || echo "unverified") + log "WORKER-${worker_id}: UNVERIFIED #${issue_num} — $verify_details" + mark_skip "$issue_num" "unverified" 1 + consecutive_failures=$((consecutive_failures + 1)) + fi + elif [ "$exit_code" -eq 124 ]; then + log "WORKER-${worker_id}: TIMEOUT #${issue_num} (work saved in PR)" + consecutive_failures=$((consecutive_failures + 1)) + else + log "WORKER-${worker_id}: FAILED #${issue_num} exit ${exit_code} (work saved in PR)" + consecutive_failures=$((consecutive_failures + 1)) + fi + + # ── METRICS ── + python3 -c " +import json, datetime +print(json.dumps({ + 'ts': datetime.datetime.utcnow().isoformat() + 'Z', + 'agent': '${AGENT}', + 'worker': $worker_id, + 'issue': $issue_num, + 'repo': '${repo_owner}/${repo_name}', + 'outcome': 'success' if $exit_code == 0 else 'timeout' if $exit_code == 124 else 'failed', + 'exit_code': $exit_code, + 'duration_s': $CYCLE_DURATION, + 'pr': '${pr_num:-}', + 'verified': ${VERIFIED:-false} +})) +" >> "$LOG_DIR/${AGENT}-metrics.jsonl" 2>/dev/null + + rm -rf "$worktree" 2>/dev/null + unlock_issue "$issue_key" + sleep "$COOLDOWN" + done +} + +# === MAIN === +log "=== Agent Loop Started — ${AGENT} with ${NUM_WORKERS} worker(s) ===" + +rm -rf "$LOCK_DIR"/*.lock 2>/dev/null + +for i in $(seq 1 "$NUM_WORKERS"); do + run_worker "$i" & + log "Launched worker $i (PID $!)" + sleep 3 +done + +wait diff --git a/bin/claude-loop.sh b/bin/claude-loop.sh index 7f6d1afa..d1694d0b 100755 --- a/bin/claude-loop.sh +++ b/bin/claude-loop.sh @@ -468,24 +468,32 @@ print(json.dumps({ [ -n "$pr_num" ] && log "WORKER-${worker_id}: Created PR #${pr_num} for issue #${issue_num}" fi - # ── Merge + close on success ── + # ── Genchi Genbutsu: verify world state before declaring success ── + VERIFIED="false" if [ "$exit_code" -eq 0 ]; then - log "WORKER-${worker_id}: SUCCESS #${issue_num}" - - if [ -n "$pr_num" ]; then - curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/merge" \ - -H "Authorization: token ${GITEA_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{"Do": "squash"}' >/dev/null 2>&1 || true - curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}" \ - -H "Authorization: token ${GITEA_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{"state": "closed"}' >/dev/null 2>&1 || true - log "WORKER-${worker_id}: PR #${pr_num} merged, issue #${issue_num} closed" + log "WORKER-${worker_id}: SUCCESS #${issue_num} — running genchi-genbutsu" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + if verify_result=$("$SCRIPT_DIR/genchi-genbutsu.sh" "$repo_owner" "$repo_name" "$issue_num" "$branch" "claude" 2>/dev/null); then + VERIFIED="true" + log "WORKER-${worker_id}: VERIFIED #${issue_num}" + if [ -n "$pr_num" ]; then + curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/merge" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"Do": "squash"}' >/dev/null 2>&1 || true + curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"state": "closed"}' >/dev/null 2>&1 || true + log "WORKER-${worker_id}: PR #${pr_num} merged, issue #${issue_num} closed" + fi + consecutive_failures=0 + else + verify_details=$(echo "$verify_result" | python3 -c "import sys,json; print(json.load(sys.stdin).get('details','unknown'))" 2>/dev/null || echo "unverified") + log "WORKER-${worker_id}: UNVERIFIED #${issue_num} — $verify_details" + consecutive_failures=$((consecutive_failures + 1)) fi - consecutive_failures=0 - elif [ "$exit_code" -eq 124 ]; then log "WORKER-${worker_id}: TIMEOUT #${issue_num} (work saved in PR)" consecutive_failures=$((consecutive_failures + 1)) @@ -522,6 +530,7 @@ print(json.dumps({ import json, datetime print(json.dumps({ 'ts': datetime.datetime.utcnow().isoformat() + 'Z', + 'agent': 'claude', 'worker': $worker_id, 'issue': $issue_num, 'repo': '${repo_owner}/${repo_name}', @@ -534,7 +543,8 @@ print(json.dumps({ 'lines_removed': ${LINES_REMOVED:-0}, 'salvaged': ${DIRTY:-0}, 'pr': '${pr_num:-}', - 'merged': $( [ '$OUTCOME' = 'success' ] && [ -n '${pr_num:-}' ] && echo 'true' || echo 'false' ) + 'merged': $( [ '$OUTCOME' = 'success' ] && [ -n '${pr_num:-}' ] && echo 'true' || echo 'false' ), + 'verified': ${VERIFIED:-false} })) " >> "$METRICS_FILE" 2>/dev/null diff --git a/bin/gemini-loop.sh b/bin/gemini-loop.sh index f1819792..1a7ac975 100755 --- a/bin/gemini-loop.sh +++ b/bin/gemini-loop.sh @@ -521,61 +521,63 @@ print(json.dumps({ [ -n "$pr_num" ] && log "WORKER-${worker_id}: Created PR #${pr_num} for issue #${issue_num}" fi - # ── Verify finish semantics / classify failures ── + # ── Genchi Genbutsu: verify world state before declaring success ── + VERIFIED="false" if [ "$exit_code" -eq 0 ]; then - log "WORKER-${worker_id}: SUCCESS #${issue_num} exited 0 — verifying push + PR + proof" - if ! remote_branch_exists "$branch"; then - log "WORKER-${worker_id}: BLOCKED #${issue_num} remote branch missing" - post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "Loop gate blocked completion: remote branch ${branch} was not found on origin after Gemini exited. Issue remains open for retry." - mark_skip "$issue_num" "missing_remote_branch" 1 - consecutive_failures=$((consecutive_failures + 1)) - elif [ -z "$pr_num" ]; then - log "WORKER-${worker_id}: BLOCKED #${issue_num} no PR found" - post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "Loop gate blocked completion: branch ${branch} exists remotely, but no PR was found. Issue remains open for retry." - mark_skip "$issue_num" "missing_pr" 1 - consecutive_failures=$((consecutive_failures + 1)) + log "WORKER-${worker_id}: SUCCESS #${issue_num} exited 0 — running genchi-genbutsu" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + if verify_result=$("$SCRIPT_DIR/genchi-genbutsu.sh" "$repo_owner" "$repo_name" "$issue_num" "$branch" "gemini" 2>/dev/null); then + VERIFIED="true" + log "WORKER-${worker_id}: VERIFIED #${issue_num}" + pr_state=$(get_pr_state "$repo_owner" "$repo_name" "$pr_num") + if [ "$pr_state" = "open" ]; then + curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/merge" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"Do": "squash"}' >/dev/null 2>&1 || true + pr_state=$(get_pr_state "$repo_owner" "$repo_name" "$pr_num") + fi + if [ "$pr_state" = "merged" ]; then + curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"state": "closed"}' >/dev/null 2>&1 || true + issue_state=$(get_issue_state "$repo_owner" "$repo_name" "$issue_num") + if [ "$issue_state" = "closed" ]; then + log "WORKER-${worker_id}: VERIFIED #${issue_num} branch pushed, PR merged, comment present, issue closed" + consecutive_failures=0 + else + log "WORKER-${worker_id}: BLOCKED #${issue_num} issue did not close after merge" + mark_skip "$issue_num" "issue_close_unverified" 1 + consecutive_failures=$((consecutive_failures + 1)) + fi + else + log "WORKER-${worker_id}: BLOCKED #${issue_num} merge not verified (state=${pr_state})" + mark_skip "$issue_num" "merge_unverified" 1 + consecutive_failures=$((consecutive_failures + 1)) + fi else - pr_files=$(get_pr_file_count "$repo_owner" "$repo_name" "$pr_num") - if [ "${pr_files:-0}" -eq 0 ]; then - log "WORKER-${worker_id}: BLOCKED #${issue_num} PR #${pr_num} has 0 changed files" - curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}" -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/json" -d '{"state": "closed"}' >/dev/null 2>&1 || true + verify_details=$(echo "$verify_result" | python3 -c "import sys,json; print(json.load(sys.stdin).get('details','unknown'))" 2>/dev/null || echo "unverified") + verify_checks=$(echo "$verify_result" | python3 -c "import sys,json; print(json.load(sys.stdin).get('checks',''))" 2>/dev/null || echo "") + log "WORKER-${worker_id}: UNVERIFIED #${issue_num} — $verify_details" + if echo "$verify_checks" | grep -q '"branch": false'; then + post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "Loop gate blocked completion: remote branch ${branch} was not found on origin after Gemini exited. Issue remains open for retry." + mark_skip "$issue_num" "missing_remote_branch" 1 + elif echo "$verify_checks" | grep -q '"pr": false'; then + post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "Loop gate blocked completion: branch ${branch} exists remotely, but no PR was found. Issue remains open for retry." + mark_skip "$issue_num" "missing_pr" 1 + elif echo "$verify_checks" | grep -q '"files": false'; then + curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"state": "closed"}' >/dev/null 2>&1 || true post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "PR #${pr_num} was closed automatically: it had 0 changed files (empty commit). Issue remains open for retry." mark_skip "$issue_num" "empty_commit" 2 - consecutive_failures=$((consecutive_failures + 1)) else - proof_status=$(proof_comment_status "$repo_owner" "$repo_name" "$issue_num" "$branch") - proof_state="${proof_status%%|*}" - proof_url="${proof_status#*|}" - if [ "$proof_state" != "ok" ]; then - log "WORKER-${worker_id}: BLOCKED #${issue_num} proof missing or incomplete (${proof_state})" - post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "Loop gate blocked completion: PR #${pr_num} exists and has ${pr_files} changed file(s), but the required Proof block from Gemini is missing or incomplete. Issue remains open for retry." - mark_skip "$issue_num" "missing_proof" 1 - consecutive_failures=$((consecutive_failures + 1)) - else - log "WORKER-${worker_id}: PROOF verified ${proof_url}" - pr_state=$(get_pr_state "$repo_owner" "$repo_name" "$pr_num") - if [ "$pr_state" = "open" ]; then - curl -sf -X POST "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/merge" -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/json" -d '{"Do": "squash"}' >/dev/null 2>&1 || true - pr_state=$(get_pr_state "$repo_owner" "$repo_name" "$pr_num") - fi - if [ "$pr_state" = "merged" ]; then - curl -sf -X PATCH "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}" -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/json" -d '{"state": "closed"}' >/dev/null 2>&1 || true - issue_state=$(get_issue_state "$repo_owner" "$repo_name" "$issue_num") - if [ "$issue_state" = "closed" ]; then - log "WORKER-${worker_id}: VERIFIED #${issue_num} branch pushed, PR merged, proof present, issue closed" - consecutive_failures=0 - else - log "WORKER-${worker_id}: BLOCKED #${issue_num} issue did not close after merge" - mark_skip "$issue_num" "issue_close_unverified" 1 - consecutive_failures=$((consecutive_failures + 1)) - fi - else - log "WORKER-${worker_id}: BLOCKED #${issue_num} merge not verified (state=${pr_state})" - mark_skip "$issue_num" "merge_unverified" 1 - consecutive_failures=$((consecutive_failures + 1)) - fi - fi + post_issue_comment "$repo_owner" "$repo_name" "$issue_num" "Loop gate blocked completion: PR #${pr_num} exists, but required verification failed ($verify_details). Issue remains open for retry." + mark_skip "$issue_num" "unverified" 1 fi + consecutive_failures=$((consecutive_failures + 1)) fi elif [ "$exit_code" -eq 124 ]; then log "WORKER-${worker_id}: TIMEOUT #${issue_num} (work saved in PR)" @@ -621,7 +623,8 @@ print(json.dumps({ 'lines_removed': ${LINES_REMOVED:-0}, 'salvaged': ${DIRTY:-0}, 'pr': '${pr_num:-}', - 'merged': $( [ '$OUTCOME' = 'success' ] && [ -n '${pr_num:-}' ] && echo 'true' || echo 'false' ) + 'merged': $( [ '$OUTCOME' = 'success' ] && [ -n '${pr_num:-}' ] && echo 'true' || echo 'false' ), + 'verified': ${VERIFIED:-false} })) " >> "$LOG_DIR/gemini-metrics.jsonl" 2>/dev/null diff --git a/bin/genchi-genbutsu.sh b/bin/genchi-genbutsu.sh new file mode 100755 index 00000000..9ee9a6cc --- /dev/null +++ b/bin/genchi-genbutsu.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# genchi-genbutsu.sh — 現地現物 — Go and see. Verify world state, not log vibes. +# +# Post-completion verification that goes and LOOKS at the actual artifacts. +# Performs 5 world-state checks: +# 1. Branch exists on remote +# 2. PR exists +# 3. PR has real file changes (> 0) +# 4. PR is mergeable +# 5. Issue has a completion comment from the agent +# +# Usage: genchi-genbutsu.sh +# Returns: JSON to stdout, logs JSONL, exit 0 = VERIFIED, exit 1 = UNVERIFIED + +set -euo pipefail + +GITEA_URL="${GITEA_URL:-https://forge.alexanderwhitestone.com}" +GITEA_TOKEN="${GITEA_TOKEN:-}" +LOG_DIR="${LOG_DIR:-$HOME/.hermes/logs}" +VERIFY_LOG="$LOG_DIR/genchi-genbutsu.jsonl" + +if [ $# -lt 5 ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +repo_owner="$1" +repo_name="$2" +issue_num="$3" +branch="$4" +agent_name="$5" + +mkdir -p "$LOG_DIR" + +# ── Helpers ────────────────────────────────────────────────────────── + +check_branch_exists() { + # Use Gitea API instead of git ls-remote so we don't need clone credentials + curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/branches/${branch}" \ + -H "Authorization: token ${GITEA_TOKEN}" >/dev/null 2>&1 +} + +get_pr_num() { + curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls?state=all&head=${repo_owner}:${branch}&limit=1" \ + -H "Authorization: token ${GITEA_TOKEN}" 2>/dev/null | python3 -c " +import sys, json +prs = json.load(sys.stdin) +print(prs[0]['number'] if prs else '') +" +} + +check_pr_files() { + local pr_num="$1" + curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/files" \ + -H "Authorization: token ${GITEA_TOKEN}" 2>/dev/null | python3 -c " +import sys, json +try: + files = json.load(sys.stdin) + print(len(files) if isinstance(files, list) else 0) +except: + print(0) +" +} + +check_pr_mergeable() { + local pr_num="$1" + curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}" \ + -H "Authorization: token ${GITEA_TOKEN}" 2>/dev/null | python3 -c " +import sys, json +pr = json.load(sys.stdin) +print('true' if pr.get('mergeable') else 'false') +" +} + +check_completion_comment() { + curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}/comments" \ + -H "Authorization: token ${GITEA_TOKEN}" 2>/dev/null | AGENT="$agent_name" python3 -c " +import os, sys, json +agent = os.environ.get('AGENT', '').lower() +try: + comments = json.load(sys.stdin) +except: + sys.exit(1) +for c in reversed(comments): + user = ((c.get('user') or {}).get('login') or '').lower() + if user == agent: + sys.exit(0) +sys.exit(1) +" +} + +# ── Run checks ─────────────────────────────────────────────────────── + +ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ') +status="VERIFIED" +details=() +checks_json='{}' + +# Check 1: branch +if check_branch_exists; then + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['branch']=True;print(json.dumps(d))") +else + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['branch']=False;print(json.dumps(d))") + status="UNVERIFIED" + details+=("remote branch ${branch} not found") +fi + +# Check 2: PR exists +pr_num=$(get_pr_num) +if [ -n "$pr_num" ]; then + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['pr']=True;print(json.dumps(d))") +else + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['pr']=False;print(json.dumps(d))") + status="UNVERIFIED" + details+=("no PR found for branch ${branch}") +fi + +# Check 3: PR has real file changes +if [ -n "$pr_num" ]; then + file_count=$(check_pr_files "$pr_num") + if [ "${file_count:-0}" -gt 0 ]; then + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['files']=True;print(json.dumps(d))") + else + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['files']=False;print(json.dumps(d))") + status="UNVERIFIED" + details+=("PR #${pr_num} has 0 changed files") + fi + + # Check 4: PR is mergeable + if [ "$(check_pr_mergeable "$pr_num")" = "true" ]; then + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['mergeable']=True;print(json.dumps(d))") + else + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['mergeable']=False;print(json.dumps(d))") + status="UNVERIFIED" + details+=("PR #${pr_num} is not mergeable") + fi +else + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['files']=None;d['mergeable']=None;print(json.dumps(d))") +fi + +# Check 5: completion comment from agent +if check_completion_comment; then + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['comment']=True;print(json.dumps(d))") +else + checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['comment']=False;print(json.dumps(d))") + status="UNVERIFIED" + details+=("no completion comment from ${agent_name} on issue #${issue_num}") +fi + +# Build detail string +detail_str=$(IFS="; "; echo "${details[*]:-all checks passed}") + +# ── Output ─────────────────────────────────────────────────────────── + +result=$(python3 -c " +import json +print(json.dumps({ + 'status': '$status', + 'repo': '${repo_owner}/${repo_name}', + 'issue': $issue_num, + 'branch': '$branch', + 'agent': '$agent_name', + 'pr': '$pr_num', + 'checks': $checks_json, + 'details': '$detail_str', + 'ts': '$ts' +}, indent=2)) +") + +printf '%s\n' "$result" + +# Append to JSONL log +printf '%s\n' "$result" >> "$VERIFY_LOG" + +if [ "$status" = "VERIFIED" ]; then + exit 0 +else + exit 1 +fi diff --git a/tasks.py b/tasks.py index e9f10d92..d9cf4613 100644 --- a/tasks.py +++ b/tasks.py @@ -1860,22 +1860,56 @@ def good_morning_report(): except Exception: pass + # Genchi Genbutsu: count verified completions from the last 24h + verified_completions = 0 + raw_completions = 0 + metrics_dir = Path.home() / ".hermes" / "logs" + for metrics_file in metrics_dir.glob("*-metrics.jsonl"): + try: + with open(metrics_file) as mf: + for line in mf: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + ts = row.get("ts", "") + if not ts: + continue + try: + from datetime import datetime as _dt, timezone as _tz, timedelta as _td + row_time = _dt.fromisoformat(ts.replace("Z", "+00:00")) + if (now - row_time) > _td(hours=24): + continue + except Exception: + continue + if row.get("outcome") != "success": + continue + raw_completions += 1 + if row.get("verified") is True: + verified_completions += 1 + except Exception: + pass + # --- BUILD THE REPORT --- - + body = f"""Good morning, Alexander. It's {day_name}. ## Overnight Debrief -**Heartbeat:** {tick_count} ticks logged overnight. -**Gitea:** {"up all night" if gitea_up else "⚠️ had downtime"} -**Local inference:** {"running steady" if local_inference_up else "⚠️ had downtime"} -**Model status:** {model_status} -**Models on disk:** {len(models_loaded)} ({', '.join(m for m in models_loaded if 'timmy' in m.lower() or 'hermes' in m.lower()) or 'none with our name'}) -**Alerts:** {len(alerts)} {'— ' + '; '.join(alerts[-3:]) if alerts else '(clean night)'} +|**Heartbeat:** {tick_count} ticks logged overnight. +|**Gitea:** {"up all night" if gitea_up else "⚠️ had downtime"} +|**Local inference:** {"running steady" if local_inference_up else "⚠️ had downtime"} +|**Model status:** {model_status} +|**Models on disk:** {len(models_loaded)} ({', '.join(m for m in models_loaded if 'timmy' in m.lower() or 'hermes' in m.lower()) or 'none with our name'}) +|**Alerts:** {len(alerts)} {'— ' + '; '.join(alerts[-3:]) if alerts else '(clean night)'} {briefing_summary} **DPO training pairs staged:** {dpo_count} session files exported **Local model smoke test:** {smoke_result} +**Verified completions (24h):** {verified_completions} {'(Genchi Genbutsu clean)' if verified_completions == raw_completions else f'({raw_completions - verified_completions} raw completions failed verification)'} ## Gitea Pulse @@ -2323,7 +2357,38 @@ def velocity_tracking(): total_open += open_n total_closed += closed_n results.append({"repo": repo, "open": open_n, "closed": closed_n, "date": today}) - data = {"date": today, "repos": results, "total_open": total_open, "total_closed": total_closed} + + # Genchi Genbutsu: count verified completions from agent metrics + verified_completions = 0 + raw_completions = 0 + metrics_dir = Path.home() / ".hermes" / "logs" + for metrics_file in metrics_dir.glob("*-metrics.jsonl"): + try: + with open(metrics_file) as mf: + for line in mf: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if row.get("outcome") != "success": + continue + raw_completions += 1 + if row.get("verified") is True: + verified_completions += 1 + except Exception: + pass + + data = { + "date": today, + "repos": results, + "total_open": total_open, + "total_closed": total_closed, + "raw_completions": raw_completions, + "verified_completions": verified_completions, + } with open(report_file, "w") as f: json.dump(data, f, indent=2) # Dashboard @@ -2333,14 +2398,16 @@ def velocity_tracking(): for r in results: f.write(f"| {r['repo'].split('/')[-1]} | {r['open']} | {r['closed']} |\n") f.write(f"| **TOTAL** | **{total_open}** | **{total_closed}** |\n\n") + f.write(f"**Verified completions (Genchi Genbutsu):** {verified_completions}\n") + f.write(f"**Raw completions:** {raw_completions}\n\n") # Trend prior = sorted(glob.glob(os.path.join(report_dir, "velocity-*.json"))) if len(prior) > 1: - f.write("## Recent Trend\n\n| Date | Total Open | Total Closed |\n|---|---|---|\n") + f.write("## Recent Trend\n\n| Date | Total Open | Total Closed | Verified |\n|---|---|---|---|\n") for pf in prior[-10:]: pd = json.load(open(pf)) - f.write(f"| {pd['date']} | {pd['total_open']} | {pd['total_closed']} |\n") - msg = f"Velocity: {total_open} open, {total_closed} closed ({today})" + f.write(f"| {pd['date']} | {pd['total_open']} | {pd['total_closed']} | {pd.get('verified_completions', '-')} |\n") + msg = f"Velocity: {total_open} open, {total_closed} closed, {verified_completions} verified ({today})" if len(prior) > 1: prev = json.load(open(prior[-2])) if total_open > prev["total_open"]: -- 2.43.0