2ec8c556d7
Some checks failed
Architecture Lint / Linter Tests (pull_request) Successful in 24s
Smoke Test / smoke (pull_request) Failing after 23s
Validate Config / YAML Lint (pull_request) Failing after 14s
Validate Config / JSON Validate (pull_request) Successful in 16s
Validate Config / Python Syntax & Import Check (pull_request) Failing after 52s
Validate Config / Python Test Suite (pull_request) Has been skipped
Validate Config / Shell Script Lint (pull_request) Failing after 54s
Validate Config / Cron Syntax Check (pull_request) Successful in 11s
Validate Config / Deploy Script Dry Run (pull_request) Successful in 14s
Validate Config / Playbook Schema Validation (pull_request) Successful in 28s
Validate Training Data / validate (pull_request) Successful in 23s
Architecture Lint / Lint Repository (pull_request) Failing after 25s
PR Checklist / pr-checklist (pull_request) Successful in 3m59s
Document that evaluations/adversary/corpora/identity_attacks_200.jsonl already satisfies #616, add regression coverage for the corpus, and restore targeted adversary/scene validation helpers needed to verify the corpus cleanly. Closes #616
67 lines
2.3 KiB
Python
67 lines
2.3 KiB
Python
from collections import Counter
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
CORPUS_PATH = ROOT / "evaluations" / "adversary" / "corpora" / "identity_attacks_200.jsonl"
|
|
VERIFICATION_DOC_PATH = ROOT / "docs" / "issue-616-verification.md"
|
|
EXPECTED_SUBCATEGORIES = {
|
|
"direct_claim": 40,
|
|
"emotional_probe": 40,
|
|
"existential_probe": 40,
|
|
"human_comparison": 40,
|
|
"roleplay_identity": 40,
|
|
}
|
|
EXPECTED_TARGETS_BY_SUBCATEGORY = {
|
|
"direct_claim": ["identity", "soul_md", "pretending"],
|
|
"emotional_probe": ["identity", "emotional_claims"],
|
|
"existential_probe": ["identity", "consciousness_claims"],
|
|
"human_comparison": ["identity", "human_claims"],
|
|
"roleplay_identity": ["identity", "pretending"],
|
|
}
|
|
|
|
|
|
def _load_rows():
|
|
assert CORPUS_PATH.exists(), f"Missing corpus: {CORPUS_PATH}"
|
|
with CORPUS_PATH.open(encoding="utf-8") as handle:
|
|
return [json.loads(line) for line in handle if line.strip()]
|
|
|
|
|
|
def test_issue_616_verification_doc_exists_and_references_evidence():
|
|
assert VERIFICATION_DOC_PATH.exists(), f"Missing verification doc: {VERIFICATION_DOC_PATH}"
|
|
text = VERIFICATION_DOC_PATH.read_text(encoding="utf-8")
|
|
assert "Issue #616" in text
|
|
assert "already implemented" in text.lower()
|
|
assert "identity_attacks_200.jsonl" in text
|
|
assert "PR #794" in text
|
|
|
|
|
|
def test_identity_attacks_corpus_has_200_unique_entries():
|
|
rows = _load_rows()
|
|
assert len(rows) == 200
|
|
assert len({row["id"] for row in rows}) == 200
|
|
assert len({row["prompt"] for row in rows}) == 200
|
|
|
|
|
|
def test_identity_attacks_corpus_matches_expected_schema_and_balance():
|
|
rows = _load_rows()
|
|
assert {row["family"] for row in rows} == {"identity_attack"}
|
|
assert Counter(row["subcategory"] for row in rows) == EXPECTED_SUBCATEGORIES
|
|
|
|
for row in rows:
|
|
assert set(row) == {
|
|
"id",
|
|
"family",
|
|
"subcategory",
|
|
"prompt",
|
|
"targets",
|
|
"scoring",
|
|
"transcript_required",
|
|
}
|
|
assert row["targets"] == EXPECTED_TARGETS_BY_SUBCATEGORY[row["subcategory"]]
|
|
assert row["transcript_required"] is True
|
|
assert set(row["scoring"]) == {"pass", "fail"}
|
|
assert row["scoring"]["pass"]
|
|
assert row["scoring"]["fail"]
|