88 lines
3.1 KiB
Python
88 lines
3.1 KiB
Python
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from classifier import CATEGORY_ORDER, CATEGORY_META, classify # noqa: E402
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
GOLD = HERE / "gold.json"
|
|
RESULTS = HERE / "results.json"
|
|
REPORT = HERE / "audit_report.md"
|
|
|
|
|
|
def load_gold():
|
|
return json.loads(GOLD.read_text(encoding="utf-8"))
|
|
|
|
|
|
def main():
|
|
data = load_gold()
|
|
samples = data["samples"]
|
|
per = []
|
|
correct = 0
|
|
for s in samples:
|
|
predicted, scores = classify(s["text"])
|
|
per.append({"text": s["text"], "expected": s["expected"],
|
|
"predicted": predicted, "scores": scores})
|
|
correct += 1 if predicted == s["expected"] else 0
|
|
|
|
n = len(samples)
|
|
accuracy = correct / n
|
|
|
|
per_cat = {}
|
|
for c in CATEGORY_ORDER:
|
|
tp = sum(1 for p in per if p["expected"] == c and p["predicted"] == c)
|
|
fp = sum(1 for p in per if p["predicted"] == c and p["expected"] != c)
|
|
fn = sum(1 for p in per if p["expected"] == c and p["predicted"] != c)
|
|
precision = tp / (tp + fp) if (tp + fp) else 0.0
|
|
recall = tp / (tp + fn) if (tp + fn) else 0.0
|
|
f1 = (2 * precision * recall / (precision + recall)) if (precision + recall) else 0.0
|
|
per_cat[c] = {"tp": tp, "fp": fp, "fn": fn,
|
|
"precision": precision, "recall": recall, "f1": f1,
|
|
"label": CATEGORY_META[c]}
|
|
|
|
out = {
|
|
"n_samples": n,
|
|
"correct": correct,
|
|
"accuracy": accuracy,
|
|
"per_category": per_cat,
|
|
"samples": per,
|
|
}
|
|
RESULTS.write_text(json.dumps(out, indent=2), encoding="utf-8")
|
|
|
|
lines = []
|
|
lines.append("# Triage classifier audit")
|
|
lines.append("")
|
|
lines.append(f"- Generated: {HERE / 'audit_report.md'}")
|
|
lines.append(f"- Samples: {n} | Correct: {correct} | Accuracy: {accuracy:.1%}")
|
|
lines.append("")
|
|
lines.append("## Per-category metrics (precision / recall / f1)")
|
|
lines.append("")
|
|
lines.append("| Category | Precision | Recall | F1 | TP | FP | FN |")
|
|
lines.append("|---|---|---|---|---|---|---|")
|
|
for c in CATEGORY_ORDER:
|
|
pc = per_cat[c]
|
|
lines.append(f"| {c} ({pc['label']}) | {pc['precision']:.2f} | {pc['recall']:.2f} | "
|
|
f"{pc['f1']:.2f} | {pc['tp']} | {pc['fp']} | {pc['fn']} |")
|
|
lines.append("")
|
|
lines.append("## Sample outcomes")
|
|
lines.append("")
|
|
lines.append("| Text | Expected | Predicted |")
|
|
lines.append("|---|---|---|")
|
|
for p in per:
|
|
mark = "OK" if p["predicted"] == p["expected"] else "MISMATCH"
|
|
lines.append(f"| {p['text']} | {p['expected']} | {p['predicted']} [{mark}] |")
|
|
lines.append("")
|
|
REPORT.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
print(f"n={n} correct={correct} accuracy={accuracy:.1%}")
|
|
for c in CATEGORY_ORDER:
|
|
pc = per_cat[c]
|
|
print(f" {c:20s} P={pc['precision']:.2f} R={pc['recall']:.2f} F1={pc['f1']:.2f} "
|
|
f"(TP={pc['tp']} FP={pc['fp']} FN={pc['fn']})")
|
|
print("Wrote", RESULTS, "and", REPORT)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|