Fix backfill: correct OpenMetrics timestamps (seconds not ms), safer import

- Timestamps now in unix seconds as floats (promtool requirement)
- Values as explicit floats (1.0 not 1)
- Import script stops Prometheus, writes blocks to volume, restarts
- Verification step checks both current and Feb 5 historical data

backfill.sh

@@ -2,49 +2,58 @@
 # Backfill historical git data into Prometheus
 # Run from the timmy-telemetry directory on Hermes:
 #   chmod +x backfill.sh && ./backfill.sh
-
 set -e
 
 echo "=== Timmy Telemetry Backfill ==="
-echo "Importing historical commit data into Prometheus..."
 echo ""
 
-# 1. Use promtool inside the Prometheus container to create TSDB blocks
-echo "Step 1: Creating TSDB blocks from OpenMetrics data..."
-docker cp backfill.txt timmy-prometheus:/tmp/backfill.txt
+# 1. Stop Prometheus so we can write to its TSDB safely
+echo "Step 1: Stopping Prometheus..."
+docker stop timmy-prometheus
 
-docker exec timmy-prometheus promtool tsdb create-blocks-from openmetrics \
-    /tmp/backfill.txt \
-    /prometheus
+# 2. Copy backfill file into the Prometheus data volume
+echo "Step 2: Creating TSDB blocks from historical data..."
+
+# Run promtool inside a temporary container with access to the Prometheus volume
+docker run --rm \
+    -v timmy-telemetry_prometheus-data:/prometheus \
+    -v "$(pwd)/backfill.txt:/tmp/backfill.txt:ro" \
+    prom/prometheus:latest \
+    promtool tsdb create-blocks-from openmetrics /tmp/backfill.txt /prometheus
 
 echo ""
-echo "Step 2: Reloading Prometheus to pick up new blocks..."
-# Tell Prometheus to reload (requires --web.enable-lifecycle flag, which we set)
-curl -s -X POST http://localhost:9090/-/reload || echo "Reload via API failed, restarting container..."
-
-# If reload didn't work, restart the container
-docker restart timmy-prometheus
+echo "Step 3: Restarting Prometheus..."
+docker start timmy-prometheus
 
 echo ""
-echo "Step 3: Waiting for Prometheus to come back..."
+echo "Step 4: Waiting for Prometheus to start..."
 sleep 5
 
-# Verify
-echo "Step 4: Verifying backfill..."
-RESULT=$(curl -s "http://localhost:9090/api/v1/query?query=timmy_sovereignty_score" 2>/dev/null)
-if echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(f'Current sovereignty score: {d[\"data\"][\"result\"][0][\"value\"][1]}' if d.get('data',{}).get('result') else 'No data yet')" 2>/dev/null; then
-    echo ""
-else
-    echo "Could not verify — check Grafana manually"
-fi
+# 5. Verify
+echo "Step 5: Verifying..."
+curl -s "http://localhost:9090/api/v1/query?query=timmy_sovereignty_score" 2>/dev/null | \
+    python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+r=d.get('data',{}).get('result',[])
+if r:
+    print(f'  Current sovereignty score: {r[0][\"value\"][1]}')
+else:
+    print('  WARNING: No current data')
+" 2>/dev/null || echo "  Could not query Prometheus"
 
-# Check historical range
-echo ""
-echo "Checking historical data range..."
-curl -s "http://localhost:9090/api/v1/query_range?query=timmy_sovereignty_score&start=2026-02-05T00:00:00Z&end=2026-02-06T00:00:00Z&step=86400" 2>/dev/null | \
-    python3 -c "import json,sys; d=json.load(sys.stdin); print(f'Feb 5 data: {\"YES\" if d.get(\"data\",{}).get(\"result\") else \"NO\"}')" 2>/dev/null || echo "Could not check"
+curl -s "http://localhost:9090/api/v1/query_range?query=timmy_sovereignty_score&start=2026-02-05T00:00:00Z&end=2026-02-06T23:59:59Z&step=86400" 2>/dev/null | \
+    python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+r=d.get('data',{}).get('result',[])
+if r and r[0].get('values'):
+    print(f'  Feb 5 sovereignty score: {r[0][\"values\"][0][1]} (backfill confirmed!)')
+else:
+    print('  WARNING: No historical data for Feb 5')
+" 2>/dev/null || echo "  Could not verify historical data"
 
 echo ""
 echo "=== Done ==="
-echo "Open Grafana at http://localhost:3033"
-echo "Set time range to 'Last 60 days' to see the full history."
+echo "Open Grafana: http://localhost:3033"
+echo "Set time range to 'Last 60 days' or 'Last 90 days'"