#!/usr/bin/env python3 """ Big Brain provider verification. Verifies that the Big Brain provider configured for Mac Hermes is reachable and can answer a simple prompt. Supports both: - OpenAI-compatible endpoints (`.../v1/models`, `.../v1/chat/completions`) - Raw Ollama endpoints (`/api/tags`, `/api/generate`) Refs: timmy-home #543 """ from __future__ import annotations import json import sys import time from datetime import datetime from pathlib import Path import requests from scripts.big_brain_provider import ( build_generate_payload, resolve_big_brain_provider, resolve_generate_url, resolve_models_url, ) RESULTS_PATH = Path("big_brain_verification.json") def _headers(provider: dict[str, str]) -> dict[str, str]: headers = {"Content-Type": "application/json"} api_key = provider.get("api_key", "") if api_key: headers["Authorization"] = f"Bearer {api_key}" return headers def check_models(provider: dict[str, str], timeout: int = 10) -> tuple[bool, float, list[str], int | None]: url = resolve_models_url(provider) started = time.time() try: response = requests.get(url, headers=_headers(provider), timeout=timeout) elapsed = time.time() - started models: list[str] = [] if response.status_code == 200: data = response.json() if provider["backend"] == "openai": models = [m.get("id", "") for m in data.get("data", [])] else: models = [m.get("name", "") for m in data.get("models", [])] return response.status_code == 200, elapsed, models, response.status_code except Exception: elapsed = time.time() - started return False, elapsed, [], None def test_generation(provider: dict[str, str], prompt: str = "Say READY", timeout: int = 30) -> tuple[bool, float, str, int | None]: url = resolve_generate_url(provider) payload = build_generate_payload(provider, prompt=prompt) started = time.time() try: response = requests.post(url, headers=_headers(provider), json=payload, timeout=timeout) elapsed = time.time() - started response_text = "" if response.status_code == 200: data = response.json() if provider["backend"] == "openai": response_text = ( data.get("choices", [{}])[0] .get("message", {}) .get("content", "") .strip() ) else: response_text = data.get("response", "").strip() return response.status_code == 200, elapsed, response_text, response.status_code except Exception: elapsed = time.time() - started return False, elapsed, "", None def main() -> int: provider = resolve_big_brain_provider() print("=" * 60) print("Big Brain Provider Verification") print(f"Timestamp: {datetime.now().isoformat()}") print(f"Provider: {provider['name']}") print(f"Backend: {provider['backend']}") print(f"Base URL: {provider['base_url']}") print(f"Model: {provider['model']}") print("=" * 60) print() models_ok, models_time, models, models_status = check_models(provider) print(f"Models endpoint: {'PASS' if models_ok else 'FAIL'} ({models_time:.2f}s, status={models_status})") if models: print(f"Models seen: {models}") print() gen_ok, gen_time, gen_response, gen_status = test_generation(provider) print(f"Generation endpoint: {'PASS' if gen_ok else 'FAIL'} ({gen_time:.2f}s, status={gen_status})") if gen_response: print(f"Response preview: {gen_response[:120]}") print() overall_ok = models_ok and gen_ok result = { "timestamp": datetime.now().isoformat(), "provider_name": provider["name"], "backend": provider["backend"], "base_url": provider["base_url"], "model": provider["model"], "models_ok": models_ok, "models_status": models_status, "models_time": models_time, "models": models, "generation_ok": gen_ok, "generation_status": gen_status, "generation_time": gen_time, "generation_response": gen_response[:200], "overall_ok": overall_ok, } RESULTS_PATH.write_text(json.dumps(result, indent=2)) print(f"Results saved to {RESULTS_PATH}") print(f"Overall: {'POD/PROVIDER LIVE' if overall_ok else 'PROVIDER ISSUES'}") return 0 if overall_ok else 1 if __name__ == "__main__": raise SystemExit(main())