1c82a2a560
Some checks failed
Architecture Lint / Linter Tests (pull_request) Successful in 23s
Smoke Test / smoke (pull_request) Failing after 15s
Validate Config / YAML Lint (pull_request) Failing after 15s
Validate Config / JSON Validate (pull_request) Successful in 17s
Validate Config / Python Syntax & Import Check (pull_request) Failing after 37s
Validate Config / Python Test Suite (pull_request) Has been skipped
Validate Config / Cron Syntax Check (pull_request) Successful in 11s
Validate Config / Shell Script Lint (pull_request) Failing after 49s
Validate Config / Deploy Script Dry Run (pull_request) Successful in 11s
Validate Config / Playbook Schema Validation (pull_request) Successful in 20s
Validate Training Data / validate (pull_request) Failing after 15s
Architecture Lint / Lint Repository (pull_request) Failing after 17s
PR Checklist / pr-checklist (pull_request) Failing after 2m53s
Added #!/usr/bin/env python3 to 74 Python scripts across: - bin/ (16 scripts) - scripts/ (48 scripts) - pipeline/ (3 scripts) - training/scripts/ (3 scripts) - hermes-sovereign/ (1 script) Also added executable permissions to fixed scripts. Test files excluded (run by pytest, not directly). Closes #681
97 lines
3.1 KiB
Python
Executable File
97 lines
3.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#!/usr/bin/env python3
|
|
"""
|
|
[EVAL] Model Evaluation Harness
|
|
Part of the Gemini Sovereign Infrastructure Suite.
|
|
|
|
Benchmarks GGUF models for speed and quality before deployment.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
import json
|
|
import argparse
|
|
import requests
|
|
|
|
BENCHMARK_PROMPTS = [
|
|
"Write a Python script to sort a list of dictionaries by a key.",
|
|
"Explain the concept of 'Sovereign AI' in three sentences.",
|
|
"What is the capital of France?",
|
|
"Write a short story about a robot learning to paint."
|
|
]
|
|
|
|
class ModelEval:
|
|
def __init__(self, endpoint: str):
|
|
self.endpoint = endpoint.rstrip("/")
|
|
|
|
def log(self, message: str):
|
|
print(f"[*] {message}")
|
|
|
|
def run_benchmark(self):
|
|
self.log(f"Starting benchmark for {self.endpoint}...")
|
|
results = []
|
|
|
|
for prompt in BENCHMARK_PROMPTS:
|
|
self.log(f"Testing prompt: {prompt[:30]}...")
|
|
|
|
start_time = time.time()
|
|
try:
|
|
# llama.cpp server /completion endpoint
|
|
response = requests.post(
|
|
f"{self.endpoint}/completion",
|
|
json={"prompt": prompt, "n_predict": 128},
|
|
timeout=60
|
|
)
|
|
duration = time.time() - start_time
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
content = data.get("content", "")
|
|
# Rough estimate of tokens (4 chars per token is a common rule of thumb)
|
|
tokens = len(content) / 4
|
|
tps = tokens / duration
|
|
|
|
results.append({
|
|
"prompt": prompt,
|
|
"duration": duration,
|
|
"tps": tps,
|
|
"success": True
|
|
})
|
|
else:
|
|
results.append({"prompt": prompt, "success": False, "error": response.text})
|
|
except Exception as e:
|
|
results.append({"prompt": prompt, "success": False, "error": str(e)})
|
|
|
|
self.report(results)
|
|
|
|
def report(self, results: list):
|
|
print("\n--- Evaluation Report ---")
|
|
total_tps = 0
|
|
success_count = 0
|
|
|
|
for r in results:
|
|
if r["success"]:
|
|
print(f"✅ {r['prompt'][:40]}... | {r['tps']:.2f} tok/s | {r['duration']:.2f}s")
|
|
total_tps += r["tps"]
|
|
success_count += 1
|
|
else:
|
|
print(f"❌ {r['prompt'][:40]}... | FAILED: {r['error']}")
|
|
|
|
if success_count > 0:
|
|
avg_tps = total_tps / success_count
|
|
print(f"\nAverage Performance: {avg_tps:.2f} tok/s")
|
|
else:
|
|
print("\n[FAILURE] All benchmarks failed.")
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Gemini Model Eval")
|
|
parser.add_argument("endpoint", help="llama-server endpoint (e.g. http://localhost:8080)")
|
|
args = parser.parse_args()
|
|
|
|
evaluator = ModelEval(args.endpoint)
|
|
evaluator.run_benchmark()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|