132 lines
4.1 KiB
Python
132 lines
4.1 KiB
Python
from datetime import datetime, timezone
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from scripts.autonomous_issue_creator import (
|
|
Incident,
|
|
build_incidents,
|
|
heartbeat_is_stale,
|
|
load_restart_counts,
|
|
sync_incidents,
|
|
)
|
|
|
|
|
|
class FakeGiteaClient:
|
|
def __init__(self, open_issues=None):
|
|
self._open_issues = list(open_issues or [])
|
|
self.created = []
|
|
self.commented = []
|
|
|
|
def list_open_issues(self):
|
|
return list(self._open_issues)
|
|
|
|
def create_issue(self, title, body):
|
|
issue = {"number": 100 + len(self.created), "title": title, "body": body}
|
|
self.created.append(issue)
|
|
return issue
|
|
|
|
def comment_issue(self, issue_number, body):
|
|
self.commented.append({"issue_number": issue_number, "body": body})
|
|
|
|
|
|
def test_load_restart_counts_reads_only_count_files(tmp_path):
|
|
(tmp_path / "act_runner.count").write_text("4\n")
|
|
(tmp_path / "worker.count").write_text("2\n")
|
|
(tmp_path / "notes.txt").write_text("ignore me")
|
|
(tmp_path / "bad.count").write_text("not-an-int")
|
|
|
|
counts = load_restart_counts(tmp_path)
|
|
|
|
assert counts == {"act_runner": 4, "worker": 2}
|
|
|
|
|
|
def test_heartbeat_is_stale_handles_missing_and_old_files(tmp_path):
|
|
now = datetime(2026, 4, 15, 4, 0, 0, tzinfo=timezone.utc)
|
|
|
|
missing = heartbeat_is_stale(tmp_path / "missing.last", now=now, max_age_seconds=900)
|
|
assert missing is True
|
|
|
|
heartbeat = tmp_path / "fleet_health.last"
|
|
heartbeat.write_text("")
|
|
old = now.timestamp() - 1800
|
|
recent = now.timestamp() - 60
|
|
|
|
heartbeat.touch()
|
|
os = __import__("os")
|
|
os.utime(heartbeat, (old, old))
|
|
assert heartbeat_is_stale(heartbeat, now=now, max_age_seconds=900) is True
|
|
|
|
os.utime(heartbeat, (recent, recent))
|
|
assert heartbeat_is_stale(heartbeat, now=now, max_age_seconds=900) is False
|
|
|
|
|
|
def test_build_incidents_captures_offline_hosts_restart_escalations_and_stale_probe():
|
|
now = datetime(2026, 4, 15, 4, 0, 0, tzinfo=timezone.utc)
|
|
failover_status = {
|
|
"timestamp": 1713148800.0,
|
|
"fleet": {"ezra": "ONLINE", "bezalel": "OFFLINE"},
|
|
}
|
|
|
|
incidents = build_incidents(
|
|
failover_status=failover_status,
|
|
restart_counts={"act_runner": 4, "worker": 2},
|
|
heartbeat_stale=True,
|
|
now=now,
|
|
restart_escalation_threshold=3,
|
|
)
|
|
|
|
fingerprints = {incident.fingerprint for incident in incidents}
|
|
assert fingerprints == {
|
|
"host-offline:bezalel",
|
|
"restart-escalation:act_runner",
|
|
"probe-stale:fleet-health",
|
|
}
|
|
|
|
titles = {incident.title for incident in incidents}
|
|
assert "[AUTO] Fleet host offline: bezalel" in titles
|
|
assert "[AUTO] Restart escalation: act_runner" in titles
|
|
assert "[AUTO] Fleet health probe stale" in titles
|
|
|
|
|
|
def test_sync_incidents_reuses_open_issues_and_creates_missing_ones():
|
|
client = FakeGiteaClient(
|
|
open_issues=[
|
|
{
|
|
"number": 71,
|
|
"title": "[AUTO] Fleet host offline: bezalel",
|
|
"body": "Fingerprint: host-offline:bezalel\n",
|
|
}
|
|
]
|
|
)
|
|
incidents = [
|
|
Incident(
|
|
fingerprint="host-offline:bezalel",
|
|
title="[AUTO] Fleet host offline: bezalel",
|
|
body="Fingerprint: host-offline:bezalel\nHost unreachable",
|
|
),
|
|
Incident(
|
|
fingerprint="probe-stale:fleet-health",
|
|
title="[AUTO] Fleet health probe stale",
|
|
body="Fingerprint: probe-stale:fleet-health\nHeartbeat missing",
|
|
),
|
|
]
|
|
|
|
results = sync_incidents(incidents, client, apply=True, comment_existing=True)
|
|
|
|
assert [result["action"] for result in results] == ["commented", "created"]
|
|
assert client.commented == [
|
|
{
|
|
"issue_number": 71,
|
|
"body": "Autonomous infrastructure detector saw the same incident again.\n\nFingerprint: host-offline:bezalel\n\nLatest evidence:\nHost unreachable",
|
|
}
|
|
]
|
|
assert client.created == [
|
|
{
|
|
"number": 100,
|
|
"title": "[AUTO] Fleet health probe stale",
|
|
"body": "Fingerprint: probe-stale:fleet-health\nHeartbeat missing",
|
|
}
|
|
]
|