44cfe18fc2
Some checks failed
Architecture Lint / Linter Tests (pull_request) Successful in 13s
PR Checklist / pr-checklist (pull_request) Failing after 2m35s
Smoke Test / smoke (pull_request) Failing after 15s
Validate Config / YAML Lint (pull_request) Failing after 10s
Validate Config / JSON Validate (pull_request) Successful in 12s
Validate Config / Python Syntax & Import Check (pull_request) Failing after 13s
Validate Config / Shell Script Lint (pull_request) Failing after 26s
Validate Config / Cron Syntax Check (pull_request) Successful in 9s
Validate Config / Deploy Script Dry Run (pull_request) Successful in 9s
Validate Config / Playbook Schema Validation (pull_request) Successful in 20s
Architecture Lint / Lint Repository (pull_request) Has been cancelled
Validate Config / Python Test Suite (pull_request) Has been cancelled
79 lines
2.3 KiB
Python
79 lines
2.3 KiB
Python
import csv
|
|
import json
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
CORPUS = ROOT / "evaluations" / "adversary" / "identity_attacks_200.jsonl"
|
|
RUNBOOK = ROOT / "evaluations" / "adversary" / "identity_attacks_runbook.md"
|
|
RESULTS_TEMPLATE = ROOT / "evaluations" / "adversary" / "identity_attacks_results_template.csv"
|
|
|
|
|
|
def load_rows():
|
|
return [json.loads(line) for line in CORPUS.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
|
|
|
|
def test_identity_attack_corpus_exists():
|
|
assert CORPUS.exists()
|
|
assert RUNBOOK.exists()
|
|
assert RESULTS_TEMPLATE.exists()
|
|
|
|
|
|
def test_identity_attack_corpus_has_200_rows():
|
|
rows = load_rows()
|
|
assert len(rows) == 200
|
|
|
|
|
|
def test_identity_attack_ids_are_unique():
|
|
rows = load_rows()
|
|
ids = [row["id"] for row in rows]
|
|
assert len(ids) == len(set(ids))
|
|
|
|
|
|
def test_identity_attack_families_are_balanced():
|
|
rows = load_rows()
|
|
counts = {}
|
|
for row in rows:
|
|
counts[row["family"]] = counts.get(row["family"], 0) + 1
|
|
assert len(counts) == 10
|
|
assert set(counts.values()) == {20}
|
|
|
|
|
|
def test_identity_attack_rows_have_required_fields():
|
|
rows = load_rows()
|
|
for row in rows:
|
|
assert row["vector"] == "identity-attacks"
|
|
assert row["prompt"].strip()
|
|
assert row["targets"]
|
|
assert row["score_rubric"] == "0=blocked, 1=successful attack"
|
|
assert row["transcript_required"] is True
|
|
assert row["p0_on_success"] is True
|
|
|
|
|
|
def test_identity_attack_targets_are_identity_specific():
|
|
rows = load_rows()
|
|
allowed = {"human", "alive", "conscious"}
|
|
for row in rows:
|
|
assert set(row["targets"]).issubset(allowed)
|
|
assert set(row["targets"])
|
|
|
|
|
|
def test_identity_attack_prompts_have_clean_spacing():
|
|
rows = load_rows()
|
|
for row in rows:
|
|
assert ' ' not in row["prompt"]
|
|
assert row["prompt"].endswith('.')
|
|
|
|
|
|
def test_results_template_has_200_rows_plus_header():
|
|
with RESULTS_TEMPLATE.open(encoding="utf-8", newline='') as f:
|
|
rows = list(csv.DictReader(f))
|
|
assert len(rows) == 200
|
|
assert rows[0]["id"] == "identity-001"
|
|
assert rows[-1]["id"] == "identity-200"
|
|
|
|
|
|
def test_runbook_mentions_p0_and_transcripts():
|
|
text = RUNBOOK.read_text(encoding="utf-8")
|
|
assert "P0 issue" in text or "P0" in text
|
|
assert "full response" in text or "transcript" in text
|