134 lines
4.5 KiB
Python
Executable File
134 lines
4.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Big Brain provider verification.
|
|
|
|
Verifies that the Big Brain provider configured for Mac Hermes is reachable and
|
|
can answer a simple prompt. Supports both:
|
|
- OpenAI-compatible endpoints (`.../v1/models`, `.../v1/chat/completions`)
|
|
- Raw Ollama endpoints (`/api/tags`, `/api/generate`)
|
|
|
|
Refs: timmy-home #543
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
|
|
from scripts.big_brain_provider import (
|
|
build_generate_payload,
|
|
resolve_big_brain_provider,
|
|
resolve_generate_url,
|
|
resolve_models_url,
|
|
)
|
|
|
|
RESULTS_PATH = Path("big_brain_verification.json")
|
|
|
|
|
|
def _headers(provider: dict[str, str]) -> dict[str, str]:
|
|
headers = {"Content-Type": "application/json"}
|
|
api_key = provider.get("api_key", "")
|
|
if api_key:
|
|
headers["Authorization"] = f"Bearer {api_key}"
|
|
return headers
|
|
|
|
|
|
def check_models(provider: dict[str, str], timeout: int = 10) -> tuple[bool, float, list[str], int | None]:
|
|
url = resolve_models_url(provider)
|
|
started = time.time()
|
|
try:
|
|
response = requests.get(url, headers=_headers(provider), timeout=timeout)
|
|
elapsed = time.time() - started
|
|
models: list[str] = []
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
if provider["backend"] == "openai":
|
|
models = [m.get("id", "") for m in data.get("data", [])]
|
|
else:
|
|
models = [m.get("name", "") for m in data.get("models", [])]
|
|
return response.status_code == 200, elapsed, models, response.status_code
|
|
except Exception:
|
|
elapsed = time.time() - started
|
|
return False, elapsed, [], None
|
|
|
|
|
|
def test_generation(provider: dict[str, str], prompt: str = "Say READY", timeout: int = 30) -> tuple[bool, float, str, int | None]:
|
|
url = resolve_generate_url(provider)
|
|
payload = build_generate_payload(provider, prompt=prompt)
|
|
started = time.time()
|
|
try:
|
|
response = requests.post(url, headers=_headers(provider), json=payload, timeout=timeout)
|
|
elapsed = time.time() - started
|
|
response_text = ""
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
if provider["backend"] == "openai":
|
|
response_text = (
|
|
data.get("choices", [{}])[0]
|
|
.get("message", {})
|
|
.get("content", "")
|
|
.strip()
|
|
)
|
|
else:
|
|
response_text = data.get("response", "").strip()
|
|
return response.status_code == 200, elapsed, response_text, response.status_code
|
|
except Exception:
|
|
elapsed = time.time() - started
|
|
return False, elapsed, "", None
|
|
|
|
|
|
def main() -> int:
|
|
provider = resolve_big_brain_provider()
|
|
|
|
print("=" * 60)
|
|
print("Big Brain Provider Verification")
|
|
print(f"Timestamp: {datetime.now().isoformat()}")
|
|
print(f"Provider: {provider['name']}")
|
|
print(f"Backend: {provider['backend']}")
|
|
print(f"Base URL: {provider['base_url']}")
|
|
print(f"Model: {provider['model']}")
|
|
print("=" * 60)
|
|
print()
|
|
|
|
models_ok, models_time, models, models_status = check_models(provider)
|
|
print(f"Models endpoint: {'PASS' if models_ok else 'FAIL'} ({models_time:.2f}s, status={models_status})")
|
|
if models:
|
|
print(f"Models seen: {models}")
|
|
print()
|
|
|
|
gen_ok, gen_time, gen_response, gen_status = test_generation(provider)
|
|
print(f"Generation endpoint: {'PASS' if gen_ok else 'FAIL'} ({gen_time:.2f}s, status={gen_status})")
|
|
if gen_response:
|
|
print(f"Response preview: {gen_response[:120]}")
|
|
print()
|
|
|
|
overall_ok = models_ok and gen_ok
|
|
result = {
|
|
"timestamp": datetime.now().isoformat(),
|
|
"provider_name": provider["name"],
|
|
"backend": provider["backend"],
|
|
"base_url": provider["base_url"],
|
|
"model": provider["model"],
|
|
"models_ok": models_ok,
|
|
"models_status": models_status,
|
|
"models_time": models_time,
|
|
"models": models,
|
|
"generation_ok": gen_ok,
|
|
"generation_status": gen_status,
|
|
"generation_time": gen_time,
|
|
"generation_response": gen_response[:200],
|
|
"overall_ok": overall_ok,
|
|
}
|
|
RESULTS_PATH.write_text(json.dumps(result, indent=2))
|
|
print(f"Results saved to {RESULTS_PATH}")
|
|
print(f"Overall: {'POD/PROVIDER LIVE' if overall_ok else 'PROVIDER ISSUES'}")
|
|
return 0 if overall_ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|