47 lines
1.5 KiB
Python
47 lines
1.5 KiB
Python
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from classifier import CATEGORY_ORDER, classify # noqa: E402
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
GOLD = HERE / "gold.json"
|
|
|
|
|
|
def main():
|
|
data = json.loads(GOLD.read_text(encoding="utf-8"))
|
|
samples = data["samples"]
|
|
|
|
correct = 0
|
|
bad = []
|
|
for s in samples:
|
|
predicted, _scores = classify(s["text"])
|
|
status = "ok" if predicted == s["expected"] else "MISMATCH"
|
|
if predicted == s["expected"]:
|
|
correct += 1
|
|
else:
|
|
bad.append((s["text"], s["expected"], predicted))
|
|
print(f"{status:8s} expected={s['expected']:20s} -> {predicted:20s} {s['text']}")
|
|
|
|
# Non-tautological guards: corpus must cover every category, and an
|
|
# out-of-vocab sentence must fall back to "other".
|
|
expected_labels = {s["expected"] for s in samples}
|
|
assert expected_labels == set(CATEGORY_ORDER), (
|
|
f"corpus must cover all five categories, got {sorted(expected_labels)}"
|
|
)
|
|
fallback = classify("hello world and goodbye today")
|
|
assert fallback[0] == "other", f"unexpected fallback: {fallback}"
|
|
assert classify("xqzjq wteu")[0] == "other"
|
|
|
|
print(f"\nALL TRIAGE CHECKS PASSED: {correct}/{len(samples)} correct")
|
|
if bad:
|
|
print("Mismatches:")
|
|
for t, e, p in bad:
|
|
print(f" {t!r}: expected {e}, got {p}")
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|