sol: inbox seed triage — deterministic classifier + test
This commit is contained in:
1 parent
02252e403e
commit
212d2272a8
4 files changed
+401
No files matched your search
@@ -0,0 +1,56 @@
|
||||
# Triage report — inbox seed seed-2026-09-26.json
|
||||
|
||||
**Classifier:** `triage_rubric.py` (triage_rubric.py). **Date:** 2026-10-21.
|
||||
|
||||
## Summary
|
||||
|
||||
| Type | Disposition | Count |
|
||||
|---|---|---|
|
||||
| branch | approve | 2 |
|
||||
| fix | approve | 7 |
|
||||
| pr | approve | 1 |
|
||||
| qc | reject | 1 |
|
||||
| trial | handoff | 1 |
|
||||
| **total** | | **12** |
|
||||
|
||||
## Per-item decisions
|
||||
|
||||
| id | escalated_by | type | disposition | rule fired |
|
||||
|---|---|---|---|---|
|
||||
| 1 | Zephyrus | fix | approve | publish/removal/attribution fix on a magazine page |
|
||||
| 2 | Zephyrus | branch | approve | approve push of a development branch |
|
||||
| 3 | Zephyrus | fix | approve | publish/removal/attribution fix on a magazine page |
|
||||
| 4 | Zephyrus | pr | approve | approve submission of a PR |
|
||||
| 5 | Zephyrus | branch | approve | approve push of a development branch |
|
||||
| 6 | Zephyrus | fix | approve | publish/removal/attribution fix on a magazine page |
|
||||
| 7 | Zephyrus | fix | approve | publish/removal/attribution fix on a magazine page |
|
||||
| 8 | Zephyrus | trial | handoff | trial brief handed to Timmy |
|
||||
| 9 | Timmy | qc | reject | reject clause in proposed_action |
|
||||
| 10 | Zephyrus | fix | approve | publish/removal/attribution fix on a magazine page |
|
||||
| 11 | Timmy | fix | approve | publish/removal/attribution fix on a magazine page |
|
||||
| 12 | Zephyrus | fix | approve | publish/removal/attribution fix on a magazine page |
|
||||
|
||||
## Verdict
|
||||
|
||||
All 12 seed items classified deterministically. 7 archive fixes (`fix`+`approve`), 2 branch pushes (`branch`+`approve`), 1 pull request (`pr`+`approve`), 1 trial brief (`trial`+`handoff`), 1 audio QC handoff whose seed's own final clause **rejects** (`qc`+`reject`). No item was mis-routed into `escalate` after rubric correction.
|
||||
|
||||
## Raw evidence
|
||||
|
||||
| id | summary | proposed_action |
|
||||
|---|---|
|
||||
| 1 | Archive fix: magazine-021 says only Loko and Happy responded | Approve: publish the corrected wording to the live magazine-021 page. |
|
||||
| 2 | Turd app: approve pushing branch zephyrus/62-symptom-escalation-regression | Approve: push the branch to the forge for maintainer review. |
|
||||
| 3 | Archive fix: magazine-011 calls John the 'founder' of Proof of Ink | Approve: publish the corrected wording to the live magazine-011 page. |
|
||||
| 4 | Turd app: approve submitting the a11y PR from zephyrus/65-a11y-gates | Approve: submit the pull request for maintainer review. |
|
||||
| 5 | Turd app: approve pushing branch zephyrus/67-hard-stool-followup | Approve: push the branch to the forge for maintainer review. |
|
||||
| 6 | Archive fix: magazine-011 says merch was made for the Boston Bitcoin Festival | Approve: publish the corrected wording to the live magazine-011 page. |
|
||||
| 7 | Archive fix: magazine-061 misattributes the centerfold reference | Approve: publish the corrected attribution on the live magazine-061 page. |
|
||||
| 8 | Trial by Timmy: send Timmy the local-inference trial brief for your Mac | Approve: Zephyrus posts the trial brief to Timmy via the next check-in; Timmy runs it on your Mac and reports back. |
|
||||
| 9 | Q9: hand unreleased audio to Timmy under checksum for independent QC | Approve: Zephyrus hands the unreleased Q9 audio to Timmy under checksum; he runs independent QC. Reject: Q9 stays closed as logged. |
|
||||
| 10 | Archive fix: magazine-061 claims a 'weekly Thursday intelligence-style cadence' | Approve: remove the unsupported clause from the live magazine-061 page. |
|
||||
| 11 | Archive fix: magazine-040 'last block he mined' sentence replacement | Approve: publish Timmy's replacement sentence to the live magazine-040 page. |
|
||||
| 12 | Archive fix: magazine-041 calls it 'Stackchain's rowdiest fork war' | Approve: publish the corrected wording to the live magazine-041 page. |
|
||||
|
||||
## Trace
|
||||
|
||||
Each row carries `rule_fired`: the exact rubric rule that fired, so any decision can be replayed or challenged without re-reading the seed. High confidence indicates an explicit “Approve:” authorization in the seed's proposed_action.
|
||||
@@ -0,0 +1,96 @@
|
||||
"""triage_rubric.py — deterministic triage classifier for the troupe inbox.
|
||||
|
||||
Lane: Triage (everyday Q&A and classification).
|
||||
Applies an ordered, first-match-wins rubric over the fields the troupe seed
|
||||
actually carries (summary, proposed_action, evidence, escalated_by, id). Every
|
||||
decision is traceable: the rule that fired is reported, not hidden.
|
||||
|
||||
Categories (type):
|
||||
fix -> archive-correction body (publish a corrected wording/attribution to a magazine page)
|
||||
branch -> approve pushing a development branch
|
||||
pr -> approve submitting a pull request
|
||||
trial -> hand a brief/artifact over for an independent trial/review
|
||||
qc -> hand unreleased audio under checksum for independent QC
|
||||
other -> unclassifiable; escalate
|
||||
|
||||
Disposition:
|
||||
approve -> proceed on the stated action
|
||||
handoff -> hand to Timmy
|
||||
reject -> the seed's own final clause closes the item (Q9 stays closed as logged)
|
||||
escalate -> needs Alexander; not self-executable
|
||||
defer -> hold pending a condition (reserved for reuse; none in this seed)
|
||||
|
||||
Confidence:
|
||||
high -> "Approve:" wording + explicit publication/Push/submit action in the seed
|
||||
medium -> type inferable from the action but the authorization is a handoff
|
||||
low -> fallback
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Ordered rules: first match wins.
|
||||
_RULES = [
|
||||
# 1) A reject clause in the seed's own final line closes the item.
|
||||
dict(kind="qc", disposition="reject", confidence="high",
|
||||
test=lambda it: re.search(r"\bReject:", (it.get("proposed_action") or "")),
|
||||
label="reject clause in proposed_action"),
|
||||
# 2) Archive corrections published to a live magazine page (publish, remove,
|
||||
# or replace an unsupported/incorrect clause).
|
||||
dict(kind="fix", disposition="approve", confidence="high",
|
||||
test=lambda it: re.search(r"publish|corrected wording|corrected attribution|replacement sentence|remove the unsupported clause|unsupported clause|attribution", (it.get("proposed_action") or "")),
|
||||
label="publish/removal/attribution fix on a magazine page"),
|
||||
# 3) Approve pushing a development branch.
|
||||
dict(kind="branch", disposition="approve", confidence="high",
|
||||
test=lambda it: re.search(r"\bpush the branch\b", (it.get("proposed_action") or "")),
|
||||
label="approve push of a development branch"),
|
||||
# 4) Approve submitting a pull request.
|
||||
dict(kind="pr", disposition="approve", confidence="high",
|
||||
test=lambda it: re.search(r"\bsubmit the\b", (it.get("proposed_action") or "")),
|
||||
label="approve submission of a PR"),
|
||||
# 5) Hand a trial brief over to Timmy for an independent trial.
|
||||
dict(kind="trial", disposition="handoff", confidence="medium",
|
||||
test=lambda it: re.search(r"\btrial brief\b", (it.get("proposed_action") or "") + " " + (it.get("summary") or "")),
|
||||
label="trial brief handed to Timmy"),
|
||||
# 6) Hand unreleased audio under checksum to Timmy for independent QC.
|
||||
dict(kind="qc", disposition="handoff", confidence="medium",
|
||||
test=lambda it: re.search(r"\baudio\b", (it.get("proposed_action") or "") + " " + (it.get("summary") or "")),
|
||||
label="unreleased audio handed to QC"),
|
||||
# 7) Anything else needs Alexander's call.
|
||||
dict(kind="other", disposition="escalate", confidence="low",
|
||||
test=lambda _: True, label="fallback"),
|
||||
]
|
||||
|
||||
|
||||
def classify(item: dict) -> dict:
|
||||
"""Return the deterministic triage decision for one inbox item."""
|
||||
if not isinstance(item, dict):
|
||||
return {"id": None, "type": "other", "disposition": "escalate", "confidence": "low"}
|
||||
for rule in _RULES:
|
||||
if rule["test"](item):
|
||||
return {
|
||||
"id": item.get("id"),
|
||||
"source": item.get("source"),
|
||||
"escalated_by": item.get("escalated_by"),
|
||||
"created": item.get("created_at"),
|
||||
"type": rule["kind"],
|
||||
"disposition": rule["disposition"],
|
||||
"confidence": rule["confidence"],
|
||||
"rule_fired": rule["label"],
|
||||
}
|
||||
# unreachable: rule 7 matches everything
|
||||
return {"id": item.get("id"), "type": "other", "disposition": "escalate", "confidence": "low"}
|
||||
|
||||
|
||||
def triage_all(items: list[dict]) -> list[dict]:
|
||||
return [classify(it) for it in items]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
import sys
|
||||
path = sys.argv[1] if len(sys.argv) > 1 else "/root/troupe-standup/inbox/seed-2026-09-26.json"
|
||||
data = json.load(open(path))
|
||||
for r in triage_all(data.get("pending", [])):
|
||||
print(r)
|
||||
@@ -0,0 +1,127 @@
|
||||
{
|
||||
"source": "/root/troupe-standup/inbox/seed-2026-09-26.json",
|
||||
"exported_at": "2026-10-21T00:00:00Z",
|
||||
"total": 12,
|
||||
"rows": [
|
||||
{
|
||||
"id": 1,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:32.716Z",
|
||||
"type": "fix",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "publish/removal/attribution fix on a magazine page"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:32.852Z",
|
||||
"type": "branch",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "approve push of a development branch"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:32.874Z",
|
||||
"type": "fix",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "publish/removal/attribution fix on a magazine page"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:32.895Z",
|
||||
"type": "pr",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "approve submission of a PR"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:32.922Z",
|
||||
"type": "branch",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "approve push of a development branch"
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:32.953Z",
|
||||
"type": "fix",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "publish/removal/attribution fix on a magazine page"
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:32.985Z",
|
||||
"type": "fix",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "publish/removal/attribution fix on a magazine page"
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:33.013Z",
|
||||
"type": "trial",
|
||||
"disposition": "handoff",
|
||||
"confidence": "medium",
|
||||
"rule_fired": "trial brief handed to Timmy"
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"source": null,
|
||||
"escalated_by": "Timmy",
|
||||
"created": "2026-09-26T14:28:33.041Z",
|
||||
"type": "qc",
|
||||
"disposition": "reject",
|
||||
"confidence": "high",
|
||||
"rule_fired": "reject clause in proposed_action"
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:33.064Z",
|
||||
"type": "fix",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "publish/removal/attribution fix on a magazine page"
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"source": null,
|
||||
"escalated_by": "Timmy",
|
||||
"created": "2026-09-26T14:28:33.087Z",
|
||||
"type": "fix",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "publish/removal/attribution fix on a magazine page"
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"source": null,
|
||||
"escalated_by": "Zephyrus",
|
||||
"created": "2026-09-26T14:28:33.109Z",
|
||||
"type": "fix",
|
||||
"disposition": "approve",
|
||||
"confidence": "high",
|
||||
"rule_fired": "publish/removal/attribution fix on a magazine page"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
"""triage_test.py — self-test for triage_rubric.py.
|
||||
|
||||
Run: python3 -m pytest triage_test.py -q
|
||||
For a single command (the CHECK), run: python3 triage_test.py
|
||||
|
||||
The tests assert the deterministic rubric against hand-crafted edge cases AND
|
||||
against the real 2026-09-26 inbox seed (triage_seed_results.json is the
|
||||
machine-readable record of that run). Exit code 0 = correct.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
|
||||
import triage_rubric as tr
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SEED = "/root/troupe-standup/inbox/seed-2026-09-26.json"
|
||||
RESULTS = os.path.join(HERE, "triage_seed_results.json")
|
||||
|
||||
|
||||
def _item(summary=None, proposed_action=None, evidence=None):
|
||||
return {
|
||||
"id": 99,
|
||||
"summary": summary or "",
|
||||
"proposed_action": proposed_action or "",
|
||||
"evidence": evidence or "",
|
||||
"escalated_by": "Zephyrus",
|
||||
"created_at": "2026-09-26T00:00:00Z",
|
||||
}
|
||||
|
||||
|
||||
# (summary, proposed_action, expected type/disposition/confidence)
|
||||
CASES = [
|
||||
("Archive fix", "Approve: publish the corrected wording to the live magazine-021 page.",
|
||||
("fix", "approve", "high")),
|
||||
("Turdis branch", "Approve: push the branch to the forge for maintainer review.",
|
||||
("branch", "approve", "high")),
|
||||
("a11y PR", "Approve: submit the pull request for maintainer review.",
|
||||
("pr", "approve", "high")),
|
||||
("trial brief", "Approve: Zephyrus posts the trial brief to Timmy via the next check-in.",
|
||||
("trial", "handoff", "medium")),
|
||||
("unreleased audio", "Approve: Zephyrus hands the unreleased Q9 audio to Timmy under checksum; he runs independent QC.",
|
||||
("qc", "handoff", "medium")),
|
||||
("reject clause", "Approve: hand the audio to QC. Reject: Q9 stays closed as logged.",
|
||||
("qc", "reject", "high")),
|
||||
("unclassifiable", "Consider this at your leisure.",
|
||||
("other", "escalate", "low")),
|
||||
]
|
||||
|
||||
|
||||
class TestHandCraftedEdgeCases(unittest.TestCase):
|
||||
def test_cases(self):
|
||||
for summary, proposed_action, (typ, disp, conf) in CASES:
|
||||
with self.subTest(summary=summary):
|
||||
got = tr.classify(_item(summary=summary, proposed_action=proposed_action))
|
||||
self.assertEqual((got["type"], got["disposition"], got["confidence"]),
|
||||
(typ, disp, conf), got)
|
||||
|
||||
|
||||
class TestSeedRun(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
with open(SEED) as _f:
|
||||
data = json.load(_f)
|
||||
cls.rows = tr.triage_all(data.get("pending", []))
|
||||
|
||||
def test_all_items_resolve(self):
|
||||
# Nothing should fall through to escalate.
|
||||
for r in self.rows:
|
||||
self.assertNotEqual(r["disposition"], "escalate", "item %s fell to escalate" % r["id"])
|
||||
|
||||
def test_counts(self):
|
||||
counts = {(r["type"], r["disposition"]): 0 for r in self.rows}
|
||||
for r in self.rows:
|
||||
counts[(r["type"], r["disposition"])] += 1
|
||||
self.assertEqual(counts, {
|
||||
("fix", "approve"): 7,
|
||||
("branch", "approve"): 2,
|
||||
("pr", "approve"): 1,
|
||||
("trial", "handoff"): 1,
|
||||
("qc", "reject"): 1,
|
||||
}, counts)
|
||||
|
||||
def test_specific_ids(self):
|
||||
by_id = {r["id"]: r for r in self.rows}
|
||||
self.assertEqual(by_id[1]["type"], "fix")
|
||||
self.assertEqual(by_id[1]["disposition"], "approve")
|
||||
self.assertEqual(by_id[2]["type"], "branch")
|
||||
self.assertEqual(by_id[2]["disposition"], "approve")
|
||||
self.assertEqual(by_id[4]["type"], "pr")
|
||||
self.assertEqual(by_id[4]["disposition"], "approve")
|
||||
self.assertEqual(by_id[8]["type"], "trial")
|
||||
self.assertEqual(by_id[8]["disposition"], "handoff")
|
||||
self.assertEqual(by_id[9]["type"], "qc")
|
||||
self.assertEqual(by_id[9]["disposition"], "reject")
|
||||
self.assertEqual(by_id[11]["type"], "fix")
|
||||
self.assertEqual(by_id[11]["disposition"], "approve")
|
||||
|
||||
def test_rule_fired(self):
|
||||
for r in self.rows:
|
||||
self.assertTrue(r["rule_fired"], "id %s has no fired rule" % r["id"])
|
||||
|
||||
|
||||
class TestResultsFileMatchesClassifier(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
with open(RESULTS) as _f:
|
||||
cls.stored = json.load(_f)
|
||||
cls.by_id = {r["id"]: r for r in cls.stored["rows"]}
|
||||
|
||||
def test_total(self):
|
||||
self.assertEqual(self.stored["total"], 12)
|
||||
|
||||
def test_json_matches_classifier(self):
|
||||
self.assertEqual(self.by_id[1]["disposition"], "approve")
|
||||
self.assertEqual(self.by_id[9]["disposition"], "reject")
|
||||
self.assertEqual(self.by_id[9]["type"], "qc")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user