Timmy Time
1fa6c3bad1
Co-authored-by: Timmy Time <timmy@alexanderwhitestone.ai> Co-committed-by: Timmy Time <timmy@alexanderwhitestone.ai>
104 lines
4.0 KiB
Python
104 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Tests for claim_annotator.py — verifies source distinction is present."""
|
|
|
|
import sys
|
|
import os
|
|
import json
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
|
|
|
from timmy.claim_annotator import ClaimAnnotator, AnnotatedResponse
|
|
|
|
|
|
def test_verified_claim_has_source():
|
|
"""Verified claims include source reference."""
|
|
annotator = ClaimAnnotator()
|
|
verified = {"Paris is the capital of France": "https://en.wikipedia.org/wiki/Paris"}
|
|
response = "Paris is the capital of France. It is a beautiful city."
|
|
|
|
result = annotator.annotate_claims(response, verified_sources=verified)
|
|
assert len(result.claims) > 0
|
|
verified_claims = [c for c in result.claims if c.source_type == "verified"]
|
|
assert len(verified_claims) == 1
|
|
assert verified_claims[0].source_ref == "https://en.wikipedia.org/wiki/Paris"
|
|
assert "[V]" in result.rendered_text
|
|
assert "[source:" in result.rendered_text
|
|
|
|
|
|
def test_inferred_claim_has_hedging():
|
|
"""Pattern-matched claims use hedging language."""
|
|
annotator = ClaimAnnotator()
|
|
response = "The weather is nice today. It might rain tomorrow."
|
|
|
|
result = annotator.annotate_claims(response)
|
|
inferred_claims = [c for c in result.claims if c.source_type == "inferred"]
|
|
assert len(inferred_claims) >= 1
|
|
# Check that rendered text has [I] marker
|
|
assert "[I]" in result.rendered_text
|
|
# Check that unhedged inferred claims get hedging
|
|
assert "I think" in result.rendered_text or "I believe" in result.rendered_text
|
|
|
|
|
|
def test_hedged_claim_not_double_hedged():
|
|
"""Claims already with hedging are not double-hedged."""
|
|
annotator = ClaimAnnotator()
|
|
response = "I think the sky is blue. It is a nice day."
|
|
|
|
result = annotator.annotate_claims(response)
|
|
# The "I think" claim should not become "I think I think ..."
|
|
assert "I think I think" not in result.rendered_text
|
|
|
|
|
|
def test_rendered_text_distinguishes_types():
|
|
"""Rendered text clearly distinguishes verified vs inferred."""
|
|
annotator = ClaimAnnotator()
|
|
verified = {"Earth is round": "https://science.org/earth"}
|
|
response = "Earth is round. Stars are far away."
|
|
|
|
result = annotator.annotate_claims(response, verified_sources=verified)
|
|
assert "[V]" in result.rendered_text # verified marker
|
|
assert "[I]" in result.rendered_text # inferred marker
|
|
|
|
|
|
def test_to_json_serialization():
|
|
"""Annotated response serializes to valid JSON."""
|
|
annotator = ClaimAnnotator()
|
|
response = "Test claim."
|
|
result = annotator.annotate_claims(response)
|
|
json_str = annotator.to_json(result)
|
|
parsed = json.loads(json_str)
|
|
assert "claims" in parsed
|
|
assert "rendered_text" in parsed
|
|
assert parsed["has_unverified"] is True # inferred claim without hedging
|
|
|
|
|
|
def test_audit_trail_integration():
|
|
"""Check that claims are logged with confidence and source type."""
|
|
# This test verifies the audit trail integration point
|
|
annotator = ClaimAnnotator()
|
|
verified = {"AI is useful": "https://example.com/ai"}
|
|
response = "AI is useful. It can help with tasks."
|
|
|
|
result = annotator.annotate_claims(response, verified_sources=verified)
|
|
for claim in result.claims:
|
|
assert claim.source_type in ("verified", "inferred")
|
|
assert claim.confidence in ("high", "medium", "low", "unknown")
|
|
if claim.source_type == "verified":
|
|
assert claim.source_ref is not None
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test_verified_claim_has_source()
|
|
print("✓ test_verified_claim_has_source passed")
|
|
test_inferred_claim_has_hedging()
|
|
print("✓ test_inferred_claim_has_hedging passed")
|
|
test_hedged_claim_not_double_hedged()
|
|
print("✓ test_hedged_claim_not_double_hedged passed")
|
|
test_rendered_text_distinguishes_types()
|
|
print("✓ test_rendered_text_distinguishes_types passed")
|
|
test_to_json_serialization()
|
|
print("✓ test_to_json_serialization passed")
|
|
test_audit_trail_integration()
|
|
print("✓ test_audit_trail_integration passed")
|
|
print("\nAll tests passed!")
|