timmy-config/ansible/scripts/validate_config.py

#!/usr/bin/env python3
"""
Config Validator — The Timmy Foundation
Validates wizard configs against golden state rules.
Run before any config deploy to catch violations early.

Usage:
    python3 validate_config.py <config_file>
    python3 validate_config.py --all  # Validate all wizard configs

Exit codes:
    0 — All validations passed
    1 — Validation errors found
    2 — File not found or parse error
"""

import sys
import os
import yaml
import fnmatch
from pathlib import Path

# === BANNED PROVIDERS — HARD POLICY ===
BANNED_PROVIDERS = {"anthropic", "claude"}
BANNED_MODEL_PATTERNS = [
    "claude-*",
    "anthropic/*",
    "*sonnet*",
    "*opus*",
    "*haiku*",
]

# === REQUIRED FIELDS ===
REQUIRED_FIELDS = {
    "model": ["default", "provider"],
    "fallback_providers": None,  # Must exist as a list
}


def is_banned_model(model_name: str) -> bool:
    """Check if a model name matches any banned pattern."""
    model_lower = model_name.lower()
    for pattern in BANNED_MODEL_PATTERNS:
        if fnmatch.fnmatch(model_lower, pattern):
            return True
    return False


def validate_config(config_path: str) -> list[str]:
    """Validate a wizard config file. Returns list of error strings."""
    errors = []

    try:
        with open(config_path) as f:
            cfg = yaml.safe_load(f)
    except FileNotFoundError:
        return [f"File not found: {config_path}"]
    except yaml.YAMLError as e:
        return [f"YAML parse error: {e}"]

    if not cfg:
        return ["Config file is empty"]

    # Check required fields
    for section, fields in REQUIRED_FIELDS.items():
        if section not in cfg:
            errors.append(f"Missing required section: {section}")
        elif fields:
            for field in fields:
                if field not in cfg[section]:
                    errors.append(f"Missing required field: {section}.{field}")

    # Check default provider
    default_provider = cfg.get("model", {}).get("provider", "")
    if default_provider.lower() in BANNED_PROVIDERS:
        errors.append(f"BANNED default provider: {default_provider}")

    default_model = cfg.get("model", {}).get("default", "")
    if is_banned_model(default_model):
        errors.append(f"BANNED default model: {default_model}")

    # Check fallback providers
    for i, fb in enumerate(cfg.get("fallback_providers", [])):
        provider = fb.get("provider", "")
        model = fb.get("model", "")

        if provider.lower() in BANNED_PROVIDERS:
            errors.append(f"BANNED fallback provider [{i}]: {provider}")

        if is_banned_model(model):
            errors.append(f"BANNED fallback model [{i}]: {model}")

    # Check providers section
    for name, provider_cfg in cfg.get("providers", {}).items():
        if name.lower() in BANNED_PROVIDERS:
            errors.append(f"BANNED provider in providers section: {name}")

        base_url = str(provider_cfg.get("base_url", ""))
        if "anthropic" in base_url.lower():
            errors.append(f"BANNED URL in provider {name}: {base_url}")

    # Check system prompt for banned references
    prompt = cfg.get("system_prompt_suffix", "")
    if isinstance(prompt, str):
        for banned in BANNED_PROVIDERS:
            if banned in prompt.lower():
                errors.append(f"BANNED provider referenced in system_prompt_suffix: {banned}")

    return errors


def main():
    if len(sys.argv) < 2:
        print(f"Usage: {sys.argv[0]} <config_file> [--all]")
        sys.exit(2)

    if sys.argv[1] == "--all":
        # Validate all wizard configs in the repo
        repo_root = Path(__file__).parent.parent.parent
        wizard_dir = repo_root / "wizards"
        all_errors = {}

        for wizard_path in sorted(wizard_dir.iterdir()):
            config_file = wizard_path / "config.yaml"
            if config_file.exists():
                errors = validate_config(str(config_file))
                if errors:
                    all_errors[wizard_path.name] = errors

        if all_errors:
            print("VALIDATION FAILED:")
            for wizard, errors in all_errors.items():
                print(f"\n  {wizard}:")
                for err in errors:
                    print(f"    - {err}")
            sys.exit(1)
        else:
            print("All wizard configs passed validation.")
            sys.exit(0)
    else:
        config_path = sys.argv[1]
        errors = validate_config(config_path)

        if errors:
            print(f"VALIDATION FAILED for {config_path}:")
            for err in errors:
                print(f"  - {err}")
            sys.exit(1)
        else:
            print(f"PASSED: {config_path}")
            sys.exit(0)


if __name__ == "__main__":
    main()