f563f76eab
Add SOURCE DISTINCTION instructions to both lite and full system prompts, requiring Timmy to cite grounded sources (memory/retrieval) and hedge appropriately when inferring. Label memory context as "GROUNDED CONTEXT" so the model can distinguish retrieved facts from pattern-matching. Fixes #463 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
94 lines
3.1 KiB
Python
94 lines
3.1 KiB
Python
from timmy.prompts import STATUS_PROMPT, SYSTEM_PROMPT, get_system_prompt
|
|
|
|
|
|
def test_system_prompt_not_empty():
|
|
assert SYSTEM_PROMPT.strip()
|
|
|
|
|
|
def test_system_prompt_no_persona_identity():
|
|
"""System prompt should NOT contain persona identity references."""
|
|
prompt = SYSTEM_PROMPT.lower()
|
|
assert "sir, affirmative" not in prompt
|
|
assert "christian" not in prompt
|
|
assert "bitcoin" not in prompt
|
|
|
|
|
|
def test_system_prompt_references_local():
|
|
assert "local" in SYSTEM_PROMPT.lower()
|
|
|
|
|
|
def test_system_prompt_is_multiline():
|
|
assert "\n" in SYSTEM_PROMPT
|
|
|
|
|
|
def test_status_prompt_not_empty():
|
|
assert STATUS_PROMPT.strip()
|
|
|
|
|
|
def test_status_prompt_no_persona():
|
|
"""Status prompt should not reference a persona."""
|
|
assert "Timmy" not in STATUS_PROMPT
|
|
|
|
|
|
def test_prompts_are_distinct():
|
|
assert SYSTEM_PROMPT != STATUS_PROMPT
|
|
|
|
|
|
def test_get_system_prompt_injects_model_name():
|
|
"""System prompt should inject actual model name from config."""
|
|
prompt = get_system_prompt(tools_enabled=False)
|
|
# Should contain the model name from settings, not the placeholder
|
|
assert "{model_name}" not in prompt
|
|
assert "llama3.1" in prompt or "qwen" in prompt
|
|
|
|
|
|
def test_full_prompt_brevity_first():
|
|
"""Full prompt should front-load brevity instructions before other content."""
|
|
prompt = get_system_prompt(tools_enabled=True)
|
|
brevity_pos = prompt.find("BREVITY")
|
|
tool_pos = prompt.find("TOOL USAGE")
|
|
memory_pos = prompt.find("MEMORY")
|
|
# Brevity section must appear before tools and memory
|
|
assert brevity_pos != -1, "Full prompt must contain BREVITY section"
|
|
assert brevity_pos < tool_pos, "Brevity must come before tool usage"
|
|
assert brevity_pos < memory_pos, "Brevity must come before memory"
|
|
|
|
|
|
def test_full_prompt_no_markdown_headers():
|
|
"""Full prompt should not use markdown headers (## / ###) that teach
|
|
the model to respond in markdown."""
|
|
prompt = get_system_prompt(tools_enabled=True)
|
|
for line in prompt.splitlines():
|
|
stripped = line.strip()
|
|
assert not stripped.startswith("## "), f"Full prompt uses markdown header: {stripped!r}"
|
|
assert not stripped.startswith("### "), (
|
|
f"Full prompt uses markdown sub-header: {stripped!r}"
|
|
)
|
|
|
|
|
|
def test_full_prompt_plain_text_brevity():
|
|
"""Full prompt should explicitly instruct plain text output."""
|
|
prompt = get_system_prompt(tools_enabled=True).lower()
|
|
assert "plain text" in prompt
|
|
|
|
|
|
def test_lite_prompt_brevity():
|
|
"""Lite prompt should also instruct brevity."""
|
|
prompt = get_system_prompt(tools_enabled=False).lower()
|
|
assert "brief" in prompt
|
|
assert "plain text" in prompt or "not markdown" in prompt
|
|
|
|
|
|
def test_full_prompt_source_distinction():
|
|
"""Full prompt must include source distinction instructions (SOUL.md)."""
|
|
prompt = get_system_prompt(tools_enabled=True)
|
|
assert "SOURCE DISTINCTION" in prompt
|
|
assert "grounded" in prompt.lower()
|
|
assert "inferred" in prompt.lower()
|
|
|
|
|
|
def test_lite_prompt_source_distinction():
|
|
"""Lite prompt must include source distinction instructions."""
|
|
prompt = get_system_prompt(tools_enabled=False).lower()
|
|
assert "source distinction" in prompt
|