hermes-agent/tests/test_user_guidance.py

#!/usr/bin/env python3
"""
Test script for user guidance pattern analysis.

This script tests the revised approach for issue #327,
focusing on user guidance patterns rather than agent proficiency.

Issue: #327 (Revised hypothesis)
"""

import sys
import os
from pathlib import Path

# Add the tools directory to path
sys.path.insert(0, str(Path(__file__).parent.parent))

def test_user_guidance_analysis():
    """Test user guidance analysis functionality."""
    print("=== Testing User Guidance Analysis ===\n")
    
    try:
        from tools.user_guidance import UserGuidanceAnalyzer
        from hermes_state import SessionDB
        
        session_db = SessionDB()
        analyzer = UserGuidanceAnalyzer(session_db)
        
        # Get a session to analyze
        sessions = session_db.get_messages.__self__.execute_write(
            "SELECT id FROM sessions ORDER BY started_at DESC LIMIT 1"
        )
        
        if not sessions:
            print("No sessions found in database.")
            return False
        
        session_id = sessions[0][0]
        print(f"Analyzing session: {session_id}\n")
        
        analysis = analyzer.analyze_user_guidance(session_id)
        
        if "error" in analysis:
            print(f"Analysis error: {analysis['error']}")
            return False
        
        print(f"Message count: {analysis['message_count']}")
        
        print("\nPrompt Patterns:")
        for p in analysis.get("prompt_patterns", [])[:3]:
            print(f"  {p['type']}: {'✓' if p.get('success') else '✗'} ({p['length']} chars)")
        
        print("\nCorrection Patterns:")
        for c in analysis.get("correction_patterns", [])[:2]:
            print(f"  {c['error_content'][:50]}... -> {c['user_correction'][:50]}...")
        
        print("\nSuccess Metrics:")
        metrics = analysis.get("success_metrics", {})
        print(f"  Tool calls: {metrics.get('tool_calls', 0)}")
        print(f"  Success rate: {metrics.get('success_rate', 0):.0%}")
        print(f"  User corrections: {metrics.get('user_corrections', 0)}")
        
        return True
        
    except Exception as e:
        print(f"Test failed: {e}")
        return False


def test_guidance_template_creation():
    """Test guidance template creation."""
    print("\n=== Testing Guidance Template Creation ===\n")
    
    try:
        from tools.user_guidance import UserGuidanceAnalyzer, GuidanceTemplateGenerator
        from hermes_state import SessionDB
        
        session_db = SessionDB()
        analyzer = UserGuidanceAnalyzer(session_db)
        generator = GuidanceTemplateGenerator(analyzer)
        
        # Get sessions
        sessions = session_db.get_messages.__self__.execute_write(
            "SELECT id FROM sessions ORDER BY started_at DESC LIMIT 3"
        )
        
        if not sessions:
            print("No sessions found.")
            return False
        
        session_ids = [s[0] for s in sessions]
        print(f"Creating template from {len(session_ids)} sessions\n")
        
        profile = generator.create_guidance_template(
            session_ids,
            name="Test Guidance Template"
        )
        
        print(f"Profile ID: {profile.profile_id}")
        print(f"Name: {profile.name}")
        print(f"Prompt patterns: {len(profile.prompt_patterns)}")
        print(f"Correction patterns: {len(profile.correction_patterns)}")
        
        # Save the template
        from tools.user_guidance import GuidanceTemplateManager
        manager = GuidanceTemplateManager()
        path = manager.save_template(profile)
        print(f"Saved to: {path}")
        
        return True
        
    except Exception as e:
        print(f"Test failed: {e}")
        return False


def test_user_guide_generation():
    """Test user guide generation."""
    print("\n=== Testing User Guide Generation ===\n")
    
    try:
        from tools.user_guidance import UserGuidanceProfile, PromptPattern, CorrectionPattern, ContextStrategy, generate_user_guide
        
        # Create a test profile
        profile = UserGuidanceProfile(
            profile_id="test_guidance_001",
            name="Test User Guidance",
            description="Test profile for guide generation",
            prompt_patterns=[
                PromptPattern(
                    pattern_type="polite_request",
                    template="Please [action] [details]",
                    success_rate=0.85,
                    usage_count=15,
                    context_requirements=[]
                ),
                PromptPattern(
                    pattern_type="question",
                    template="How do I [action]?",
                    success_rate=0.75,
                    usage_count=20,
                    context_requirements=[]
                )
            ],
            correction_patterns=[
                CorrectionPattern(
                    error_type="file_not_found",
                    correction_strategy="direct",
                    effectiveness=0.90,
                    common_phrases=["Use the correct path: [path]", "The file is at [location]"]
                ),
                CorrectionPattern(
                    error_type="command_not_found",
                    correction_strategy="example",
                    effectiveness=0.80,
                    common_phrases=["Try: [command]", "Use [alternative] instead"]
                )
            ],
            context_strategies=[
                ContextStrategy(
                    strategy_type="file_reference",
                    description="Reference specific files",
                    effectiveness=0.85,
                    token_cost=10
                ),
                ContextStrategy(
                    strategy_type="code_example",
                    description="Provide code examples",
                    effectiveness=0.90,
                    token_cost=50
                )
            ],
            created_at="2026-04-13T00:00:00",
            source_analysis="Test sessions"
        )
        
        guide = generate_user_guide(profile)
        
        print("Generated User Guide:")
        print("=" * 50)
        print(guide[:1000] + "..." if len(guide) > 1000 else guide)
        
        return True
        
    except Exception as e:
        print(f"Test failed: {e}")
        return False


def main():
    """Run all tests."""
    print("User Guidance Pattern Analysis Test Suite")
    print("=" * 50)
    
    tests = [
        ("User Guidance Analysis", test_user_guidance_analysis),
        ("Guidance Template Creation", test_guidance_template_creation),
        ("User Guide Generation", test_user_guide_generation)
    ]
    
    results = []
    
    for name, test_func in tests:
        print(f"\nRunning: {name}")
        try:
            result = test_func()
            results.append((name, result))
            print(f"Result: {'PASS' if result else 'FAIL'}")
        except Exception as e:
            print(f"Error: {e}")
            results.append((name, False))
    
    print("\n" + "=" * 50)
    print("Test Results:")
    
    passed = sum(1 for _, result in results if result)
    total = len(results)
    
    for name, result in results:
        status = "✓ PASS" if result else "✗ FAIL"
        print(f"  {status}: {name}")
    
    print(f"\nPassed: {passed}/{total}")
    
    return 0 if passed == total else 1


if __name__ == "__main__":
    sys.exit(main())