Implement 現地現物 (Genchi Genbutsu) post-completion verification: - Add bin/genchi-genbutsu.sh performing 5 world-state checks: 1. Branch exists on remote 2. PR exists 3. PR has real file changes (> 0) 4. PR is mergeable 5. Issue has a completion comment from the agent - Wire verification into all agent loops: - bin/claude-loop.sh: call genchi-genbutsu before merge/close - bin/gemini-loop.sh: delegate existing inline checks to genchi-genbutsu - bin/agent-loop.sh: resurrect generic agent loop with genchi-genbutsu wired in - Update metrics JSONL to include 'verified' field for all loops - Update burn monitor (tasks.py velocity_tracking): - Report verified_completion count alongside raw completions - Dashboard shows verified trend history - Update morning report (tasks.py good_morning_report): - Count only verified completions from the last 24h - Surface verification failures in the report body Fixes #348 Refs #345
180 lines
6.0 KiB
Bash
Executable File
180 lines
6.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# genchi-genbutsu.sh — 現地現物 — Go and see. Verify world state, not log vibes.
|
|
#
|
|
# Post-completion verification that goes and LOOKS at the actual artifacts.
|
|
# Performs 5 world-state checks:
|
|
# 1. Branch exists on remote
|
|
# 2. PR exists
|
|
# 3. PR has real file changes (> 0)
|
|
# 4. PR is mergeable
|
|
# 5. Issue has a completion comment from the agent
|
|
#
|
|
# Usage: genchi-genbutsu.sh <repo_owner> <repo_name> <issue_num> <branch> <agent_name>
|
|
# Returns: JSON to stdout, logs JSONL, exit 0 = VERIFIED, exit 1 = UNVERIFIED
|
|
|
|
set -euo pipefail
|
|
|
|
GITEA_URL="${GITEA_URL:-https://forge.alexanderwhitestone.com}"
|
|
GITEA_TOKEN="${GITEA_TOKEN:-}"
|
|
LOG_DIR="${LOG_DIR:-$HOME/.hermes/logs}"
|
|
VERIFY_LOG="$LOG_DIR/genchi-genbutsu.jsonl"
|
|
|
|
if [ $# -lt 5 ]; then
|
|
echo "Usage: $0 <repo_owner> <repo_name> <issue_num> <branch> <agent_name>" >&2
|
|
exit 2
|
|
fi
|
|
|
|
repo_owner="$1"
|
|
repo_name="$2"
|
|
issue_num="$3"
|
|
branch="$4"
|
|
agent_name="$5"
|
|
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# ── Helpers ──────────────────────────────────────────────────────────
|
|
|
|
check_branch_exists() {
|
|
# Use Gitea API instead of git ls-remote so we don't need clone credentials
|
|
curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/branches/${branch}" \
|
|
-H "Authorization: token ${GITEA_TOKEN}" >/dev/null 2>&1
|
|
}
|
|
|
|
get_pr_num() {
|
|
curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls?state=all&head=${repo_owner}:${branch}&limit=1" \
|
|
-H "Authorization: token ${GITEA_TOKEN}" 2>/dev/null | python3 -c "
|
|
import sys, json
|
|
prs = json.load(sys.stdin)
|
|
print(prs[0]['number'] if prs else '')
|
|
"
|
|
}
|
|
|
|
check_pr_files() {
|
|
local pr_num="$1"
|
|
curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}/files" \
|
|
-H "Authorization: token ${GITEA_TOKEN}" 2>/dev/null | python3 -c "
|
|
import sys, json
|
|
try:
|
|
files = json.load(sys.stdin)
|
|
print(len(files) if isinstance(files, list) else 0)
|
|
except:
|
|
print(0)
|
|
"
|
|
}
|
|
|
|
check_pr_mergeable() {
|
|
local pr_num="$1"
|
|
curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/pulls/${pr_num}" \
|
|
-H "Authorization: token ${GITEA_TOKEN}" 2>/dev/null | python3 -c "
|
|
import sys, json
|
|
pr = json.load(sys.stdin)
|
|
print('true' if pr.get('mergeable') else 'false')
|
|
"
|
|
}
|
|
|
|
check_completion_comment() {
|
|
curl -sf "${GITEA_URL}/api/v1/repos/${repo_owner}/${repo_name}/issues/${issue_num}/comments" \
|
|
-H "Authorization: token ${GITEA_TOKEN}" 2>/dev/null | AGENT="$agent_name" python3 -c "
|
|
import os, sys, json
|
|
agent = os.environ.get('AGENT', '').lower()
|
|
try:
|
|
comments = json.load(sys.stdin)
|
|
except:
|
|
sys.exit(1)
|
|
for c in reversed(comments):
|
|
user = ((c.get('user') or {}).get('login') or '').lower()
|
|
if user == agent:
|
|
sys.exit(0)
|
|
sys.exit(1)
|
|
"
|
|
}
|
|
|
|
# ── Run checks ───────────────────────────────────────────────────────
|
|
|
|
ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
status="VERIFIED"
|
|
details=()
|
|
checks_json='{}'
|
|
|
|
# Check 1: branch
|
|
if check_branch_exists; then
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['branch']=True;print(json.dumps(d))")
|
|
else
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['branch']=False;print(json.dumps(d))")
|
|
status="UNVERIFIED"
|
|
details+=("remote branch ${branch} not found")
|
|
fi
|
|
|
|
# Check 2: PR exists
|
|
pr_num=$(get_pr_num)
|
|
if [ -n "$pr_num" ]; then
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['pr']=True;print(json.dumps(d))")
|
|
else
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['pr']=False;print(json.dumps(d))")
|
|
status="UNVERIFIED"
|
|
details+=("no PR found for branch ${branch}")
|
|
fi
|
|
|
|
# Check 3: PR has real file changes
|
|
if [ -n "$pr_num" ]; then
|
|
file_count=$(check_pr_files "$pr_num")
|
|
if [ "${file_count:-0}" -gt 0 ]; then
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['files']=True;print(json.dumps(d))")
|
|
else
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['files']=False;print(json.dumps(d))")
|
|
status="UNVERIFIED"
|
|
details+=("PR #${pr_num} has 0 changed files")
|
|
fi
|
|
|
|
# Check 4: PR is mergeable
|
|
if [ "$(check_pr_mergeable "$pr_num")" = "true" ]; then
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['mergeable']=True;print(json.dumps(d))")
|
|
else
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['mergeable']=False;print(json.dumps(d))")
|
|
status="UNVERIFIED"
|
|
details+=("PR #${pr_num} is not mergeable")
|
|
fi
|
|
else
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['files']=None;d['mergeable']=None;print(json.dumps(d))")
|
|
fi
|
|
|
|
# Check 5: completion comment from agent
|
|
if check_completion_comment; then
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['comment']=True;print(json.dumps(d))")
|
|
else
|
|
checks_json=$(echo "$checks_json" | python3 -c "import sys,json;d=json.load(sys.stdin);d['comment']=False;print(json.dumps(d))")
|
|
status="UNVERIFIED"
|
|
details+=("no completion comment from ${agent_name} on issue #${issue_num}")
|
|
fi
|
|
|
|
# Build detail string
|
|
detail_str=$(IFS="; "; echo "${details[*]:-all checks passed}")
|
|
|
|
# ── Output ───────────────────────────────────────────────────────────
|
|
|
|
result=$(python3 -c "
|
|
import json
|
|
print(json.dumps({
|
|
'status': '$status',
|
|
'repo': '${repo_owner}/${repo_name}',
|
|
'issue': $issue_num,
|
|
'branch': '$branch',
|
|
'agent': '$agent_name',
|
|
'pr': '$pr_num',
|
|
'checks': $checks_json,
|
|
'details': '$detail_str',
|
|
'ts': '$ts'
|
|
}, indent=2))
|
|
")
|
|
|
|
printf '%s\n' "$result"
|
|
|
|
# Append to JSONL log
|
|
printf '%s\n' "$result" >> "$VERIFY_LOG"
|
|
|
|
if [ "$status" = "VERIFIED" ]; then
|
|
exit 0
|
|
else
|
|
exit 1
|
|
fi
|