feat: Add automated agent lifecycle management to fleet orchestrator (#552)

scripts/fleet_orchestrator.py

@@ -0,0 +1,130 @@
+
+import subprocess
+import sys
+import os
+
+FLEET_HOSTS = os.environ.get("FLEET_HOSTS", "143.198.27.163 104.131.15.18").split()
+TIMMY_USER = os.environ.get("TIMMY_USER", "root")
+TIMMY_DIR = os.environ.get("TIMMY_HOME", "/root") + "/timmy"
+
+
+def run_remote_command(host, command):
+    """Executes a command remotely on a given host via SSH."""
+    ssh_command = ["ssh", f"{TIMMY_USER}@{host}", command]
+    print(f"Executing on {host}: {' '.join(ssh_command)}")
+    try:
+        result = subprocess.run(ssh_command, capture_output=True, text=True, check=True)
+        print(f"[{host}] STDOUT:\n{result.stdout}")
+        if result.stderr:
+            print(f"[{host}] STDERR:\n{result.stderr}")
+        return result.stdout
+    except subprocess.CalledProcessError as e:
+        print(f"[{host}] ERROR: Command failed with exit code {e.returncode}")
+        print(f"[{host}] STDOUT:\n{e.stdout}")
+        print(f"[{host}] STDERR:\n{e.stderr}")
+        return None
+    except Exception as e:
+        print(f"[{host}] AN UNEXPECTED ERROR OCCURRED: {e}")
+        return None
+
+
+def deploy_agent(host):
+    """Deploys the agent on a remote host, including its systemd service and code."""
+    print(f"Deploying agent on {host}...")
+
+    # Ensure the remote directory structure exists
+    run_remote_command(host, f"mkdir -p {TIMMY_DIR}/timmy-home/agent")
+    run_remote_command(host, f"mkdir -p /etc/systemd/system")
+
+    # Copy the timmy-agent.service file
+    subprocess.run(["scp", "configs/timmy-agent.service", f"{TIMMY_USER}@{host}:/etc/systemd/system/timmy-agent.service"], check=True)
+
+    # Copy the agent code
+    # Assuming 'agent' directory is parallel to 'scripts' within 'timmy-home'
+    subprocess.run(["scp", "-r", "agent/", f"{TIMMY_USER}@{host}:{TIMMY_DIR}/timmy-home/"], check=True)
+
+    # Reload systemd, enable and start the service
+    commands = [
+        "systemctl daemon-reload",
+        "systemctl enable timmy-agent.service",
+        "systemctl start timmy-agent.service"
+    ]
+    for cmd in commands:
+        run_remote_command(host, cmd)
+
+
+
+def retire_agent(host):
+    """Retires the agent on a remote host by stopping, disabling, and removing its systemd service and code."""
+    print(f"Retiring agent on {host}...")
+    commands = [
+        "systemctl stop timmy-agent.service",
+        "systemctl disable timmy-agent.service",
+        "rm -f /etc/systemd/system/timmy-agent.service",
+        "systemctl daemon-reload", # Reload daemon to remove the service from memory
+        f"rm -rf {TIMMY_DIR}/timmy-home/agent" # Optional: remove agent code
+    ]
+    for cmd in commands:
+        run_remote_command(host, cmd)
+
+
+def start_agent(host):
+    """Starts the timmy-agent.service on a remote host."""
+    print(f"Starting agent on {host}...")
+    run_remote_command(host, f"systemctl start timmy-agent.service")
+
+
+def stop_agent(host):
+    """Stops the timmy-agent.service on a remote host."""
+    print(f"Stopping agent on {host}...")
+    run_remote_command(host, f"systemctl stop timmy-agent.service")
+
+
+def update_agent(host):
+    """Pulls the latest timmy-home repo and restarts the agent on a remote host."""
+    print(f"Updating agent on {host}...")
+    commands = [
+        f"cd {TIMMY_DIR}/timmy-home && git pull",
+        f"systemctl restart timmy-agent.service"
+    ]
+    for cmd in commands:
+        run_remote_command(host, cmd)
+
+
+def status_agent(host):
+    """Checks the status of the timmy-agent.service on a remote host."""
+    print(f"Checking agent status on {host}...")
+    run_remote_command(host, f"systemctl status timmy-agent.service --no-pager")
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python fleet_orchestrator.py <command> [host]")
+        print("Commands: deploy, start, stop, update, status")
+        sys.exit(1)
+
+    action = sys.argv[1]
+    target_host = sys.argv[2] if len(sys.argv) > 2 else None
+
+    hosts_to_target = [target_host] if target_host else FLEET_HOSTS
+
+    for host in hosts_to_target:
+        if action == "deploy":
+            deploy_agent(host)
+        elif action == "start":
+            start_agent(host)
+        elif action == "stop":
+            stop_agent(host)
+        elif action == "update":
+            update_agent(host)
+        elif action == "status":
+            status_agent(host)
+        elif action == "retire":
+            retire_agent(host)
+        else:
+            print(f"Unknown command: {action}")
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()