57 lines
1.9 KiB
Python
57 lines
1.9 KiB
Python
#!/usr/bin/env python3
|
|
"""diag_denominators.py -- find which baseline each paper percentage actually uses.
|
|
|
|
The paper states percentages; the table holds F1. If the stated percentage does
|
|
not come out of the best-textual baseline, this brute-forces every model as the
|
|
denominator so the archive reports the real one instead of guessing.
|
|
"""
|
|
import itertools
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
BASE = os.path.dirname(os.path.abspath(__file__))
|
|
d = json.load(open(os.path.join(BASE, "claims.json"), encoding="utf-8"))
|
|
|
|
# Rebuild the numbers from the same source verify_claims.py uses.
|
|
sys.path.insert(0, BASE)
|
|
import verify_claims as vc # noqa: E402
|
|
|
|
|
|
def pct(a, b):
|
|
return (a - b) / b * 100.0
|
|
|
|
|
|
TARGETS = [
|
|
("FR", "MuRAG", 7.69), ("VS", "MuRAG", 12.29), ("TTL", "MuRAG", 29.06),
|
|
]
|
|
|
|
print("== which denominator reproduces each stated task gain? ==")
|
|
for task, num_model, target in TARGETS:
|
|
row = vc.TASK_F1[task]
|
|
num = row[num_model]
|
|
hits = []
|
|
for m, v in row.items():
|
|
got = pct(num, v)
|
|
if abs(got - target) <= 0.02:
|
|
hits.append((m, round(got, 3)))
|
|
print("%-4s stated=%6.2f%% numerator=%s(%.4f) matching denominators=%s" %
|
|
(task, target, num_model, num, hits or "NONE"))
|
|
|
|
# also show every candidate, so 'none' is visible rather than assumed
|
|
print(" all: " + ", ".join("%s=%+.2f%%" % (m, pct(num, v)) for m, v in row.items()))
|
|
|
|
print()
|
|
print("== AR: is 7.22 / 5.82 relative or absolute? ==")
|
|
fifo = vc.AR_F1["FIFO"]
|
|
for m in ("A-Mem", "MuRAG"):
|
|
rel = -pct(vc.AR_F1[m], fifo)
|
|
ab = (fifo - vc.AR_F1[m]) * 100.0
|
|
print(" %s vs FIFO: relative=%6.2f%% absolute_points=%6.2f F1=%.4f" %
|
|
(m, rel, ab, vc.AR_F1[m]))
|
|
|
|
print()
|
|
print("== Full(MM) 8.08: relative vs absolute? ==")
|
|
fm, ft = vc.OVERALL_F1["Full (MM)"], vc.OVERALL_F1["Full (Text)"]
|
|
print(" vs Full(Text): relative=%6.2f%% absolute_points=%6.2f" % (-pct(fm, ft), (ft - fm) * 100.0))
|