troupe-standup/wizard-work/sol/test_classify_fallback.py

#!/usr/bin/env python3
"""Cells for the sol classifier's fallback contract (Q119).

The defect these cells pin: an item that matched no category rule used to be
returned as `general` -- a real category, at score 0 -- so a demand the rule set
did not understand was indistinguishable from one it had understood and scored.
The contract now is `unclassified` at score 0, counted beside the three real
categories so the aggregate still sums to the item count.

The invariant is one-directional on purpose: a *matched* item may be any real
category (the taxonomy is not this file's business), but a zero-score item may
never be.  Four of Q119's five repro strings are zero-score under the current
rules; the fifth, "Triage the inbound queue before standup.", is a `general`
rule hit on `standup` at score 1 -- measured, and pinned as such in
`test_repro_with_a_rule_hit_is_classified_by_rule` rather than being forced
into the fallback to make a table look tidy.
"""
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))

import classify  # noqa: E402  (the module under test, imported after sys.path)

# Q119's five hand-written imperatives, verbatim.
REPRO = [
    "Ship the manifest to ops by Friday.",
    "Move the nightly cron to the cheaper provider tonight.",
    "We need a call on the provider tonight - reply by 17:00.",
    "Triage the inbound queue before standup.",
    "Roll back the ingress change.",
]
ZERO_SCORE_REPRO = [t for t in REPRO if "standup" not in t]


class FallbackContract(unittest.TestCase):
    def test_zero_score_always_reads_unclassified(self):
        for text in ZERO_SCORE_REPRO:
            with self.subTest(text=text):
                cat, score, reason = classify.classify(text)
                self.assertEqual(score, 0)
                self.assertEqual(
                    cat,
                    classify.UNCLASSIFIED,
                    "%r landed in %r, want %r" % (
                        text, cat, classify.UNCLASSIFIED),
                )
                self.assertIn("unclassified", reason)

    def test_zero_score_never_lands_in_a_real_category(self):
        for text in ZERO_SCORE_REPRO:
            with self.subTest(text=text):
                cat, score, _ = classify.classify(text)
                if score == 0:
                    self.assertNotIn(cat, classify.CATS)

    def test_every_repro_is_labelled_explicitly(self):
        # No string may be ambiguous: either it matched (score >= 1 and a
        # `matched` reason) or it is the explicit fallback.  A real category at
        # score 0 is the silent default Q119 filed.
        for text in REPRO:
            with self.subTest(text=text):
                cat, score, reason = classify.classify(text)
                if score == 0:
                    self.assertEqual(cat, classify.UNCLASSIFIED)
                else:
                    self.assertIn(cat, classify.CATS)
                    self.assertIn("matched", reason)

    def test_repro_with_a_rule_hit_is_classified_by_rule(self):
        cat, score, reason = classify.classify(
            "Triage the inbound queue before standup."
        )
        self.assertEqual(cat, "general")
        self.assertEqual(score, 1)
        self.assertIn("standup", reason)

    def test_unclassified_is_counted_beside_real_categories(self):
        self.assertNotIn(classify.UNCLASSIFIED, classify.CATS)
        self.assertIn(classify.UNCLASSIFIED, classify.COUNTED)
        self.assertEqual(
            sorted(classify.COUNTED),
            sorted(classify.CATS + [classify.UNCLASSIFIED]),
        )

    def test_matched_item_still_classifies(self):
        # Control: a phrase that does match a rule is not swept into the
        # fallback by the repair.  "still renders" is an archive_fix rule.
        cat, score, reason = classify.classify(
            "The page still renders a dead clone URL."
        )
        self.assertEqual(cat, "archive_fix")
        self.assertGreaterEqual(score, 1)
        self.assertIn("matched", reason)

    def test_unclassified_reason_names_no_category(self):
        _, _, reason = classify.classify(ZERO_SCORE_REPRO[0])
        for cat in classify.CATS:
            self.assertNotIn(
                cat,
                reason,
                "fallback reason claims category %r: %r" % (cat, reason),
            )

    def test_aggregate_counts_unclassified_and_sums_to_items(self):
        seed = {
            "pending": [
                {"id": 1, "evidence": "README still renders a dead URL",
                 "proposed_action": "publish release notes"},
                {"id": 2, "evidence": "Ship the manifest to ops by Friday.",
                 "proposed_action": ""},
                {"id": 3, "evidence": "Roll back the ingress change.",
                 "proposed_action": ""},
            ]
        }
        with tempfile.TemporaryDirectory() as td:
            seed_path = Path(td) / "seed.json"
            out_path = Path(td) / "out.json"
            seed_path.write_text(json.dumps(seed), encoding="utf-8")
            proc = subprocess.run(
                [sys.executable, str(HERE / "classify.py"),
                 "--seed", str(seed_path), "--output", str(out_path)],
                capture_output=True, text=True,
            )
            self.assertEqual(proc.returncode, 0, proc.stderr)
            art = json.loads(out_path.read_text(encoding="utf-8"))
        self.assertEqual(art["categories"].get("unclassified"), 2)
        self.assertEqual(sum(art["categories"].values()), len(art["items"]))
        by_id = {it["id"]: it for it in art["items"]}
        self.assertEqual(by_id[2]["category"], classify.UNCLASSIFIED)
        self.assertEqual(by_id[3]["category"], classify.UNCLASSIFIED)
        self.assertIn(by_id[1]["category"], classify.CATS)

    def test_cli_refuses_empty_seed(self):
        with tempfile.TemporaryDirectory() as td:
            seed_path = Path(td) / "seed.json"
            out_path = Path(td) / "out.json"
            seed_path.write_text(json.dumps({"pending": []}), encoding="utf-8")
            proc = subprocess.run(
                [sys.executable, str(HERE / "classify.py"),
                 "--seed", str(seed_path), "--output", str(out_path)],
                capture_output=True, text=True,
            )
        self.assertEqual(proc.returncode, 1)
        self.assertIn("no pending items", proc.stderr)

    def test_source_has_no_general_default_string(self):
        src = (HERE / "classify.py").read_text(encoding="utf-8")
        self.assertNotIn("defaulted to general", src)
        self.assertIn("unclassified", src)


if __name__ == "__main__":
    unittest.main(verbosity=2)