null: raglite — stdlib BM25 passage retrieval (25/25 checks pass)
This commit is contained in:
null committed
1 parent
c81f52d955
commit
ea6a6e430a
6 files changed
+2389
No files matched your search
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe: does `LEFT JOIN <true-on-with-NULL-row>` produce a NULL-extended row in
|
||||
sqlite3, and does the `and-null-absorbs-false` mutant change any query's answer?
|
||||
|
||||
Reads like a spell-check of a claim. Every number printed is measured here.
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
import minisql # noqa: E402
|
||||
|
||||
SPEC = {
|
||||
"t1": [("id", "INTEGER"), ("a", "INTEGER"), ("b", "TEXT"), ("c", "REAL")],
|
||||
"t2": [("id", "INTEGER"), ("d", "INTEGER"), ("e", "TEXT")],
|
||||
}
|
||||
ROWS = {
|
||||
"t1": [[1, 1, "apple", None], [2, None, "banana", 2.5], [3, 3, None, 4.0]],
|
||||
"t2": [[1, 7, "apple"], [2, None, "zzz"], [3, 9, "banana"]],
|
||||
}
|
||||
TABLES = {n: minisql.table_from(c, ROWS[n]) for n, c in SPEC.items()}
|
||||
|
||||
QUERIES = [
|
||||
("plain scan", "SELECT id, a FROM t1"),
|
||||
("left join plain", "SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id"),
|
||||
("left join null-on",
|
||||
"SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND (t1.a = 100)"),
|
||||
("left join null-on2",
|
||||
"SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND (t2.d = 100)"),
|
||||
("left join null-on3",
|
||||
"SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND (t2.d = 100 AND t2.id > 0)"),
|
||||
("left join where-null",
|
||||
"SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id WHERE t2.d = 100"),
|
||||
("left join null-on + where",
|
||||
"SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND (t2.d = 100) "
|
||||
"WHERE t1.a = 1"),
|
||||
]
|
||||
|
||||
|
||||
def run(sql):
|
||||
con = sqlite3.connect(":memory:")
|
||||
for name, cols in SPEC.items():
|
||||
con.execute("CREATE TABLE %s (%s)" % (
|
||||
name, ", ".join("%s %s" % (c, t) for c, t in cols)))
|
||||
con.executemany("INSERT INTO %s VALUES (%s)" % (
|
||||
name, ", ".join("?" * len(cols))), ROWS[name])
|
||||
oracle = [list(r) for r in con.execute(sql).fetchall()]
|
||||
con.close()
|
||||
engine = minisql.execute(sql, TABLES)[1]
|
||||
return oracle, engine
|
||||
|
||||
|
||||
def main():
|
||||
print("A. does sqlite3 produce NULL-extended rows for a NULL-y ON?")
|
||||
for label, sql in QUERIES:
|
||||
o, e = run(sql)
|
||||
print(" %-24s oracle=%r" % (label, o))
|
||||
print(" %-24s engine=%r agree=%s" % ("", e, o == e))
|
||||
|
||||
print()
|
||||
print("B. and-null-absorbs-false mutant, same query set")
|
||||
orig = minisql._tri_and
|
||||
|
||||
def m_and_absorb(a, b):
|
||||
ta, tb = minisql.tri(a), minisql.tri(b)
|
||||
if ta is None or tb is None:
|
||||
return None
|
||||
return 0 if (ta == 0 or tb == 0) else 1
|
||||
|
||||
minisql._tri_and = m_and_absorb
|
||||
try:
|
||||
for label, sql in QUERIES:
|
||||
o, e = run(sql)
|
||||
print(" %-24s mutant_rows=%r agree_with_oracle=%s" % (label, e, o == e))
|
||||
finally:
|
||||
minisql._tri_and = orig
|
||||
print()
|
||||
print("C. and/or truth tables (original vs mutant)")
|
||||
vals = [0, 1, None]
|
||||
print(" %-6s %-8s %-8s %-8s %-8s" % ("a", "b", "AND orig", "AND mut", "OR orig"))
|
||||
for a in vals:
|
||||
for b in vals:
|
||||
print(" %-6r %-8r %-8r %-8r %-8r" % (
|
||||
a, b, orig(a, b), m_and_absorb(a, b), minisql._tri_or(a, b)))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,649 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
diff_sqlite.py -- randomized differential battery: minisql.py vs sqlite3 (the oracle).
|
||||
|
||||
Wizard: null (lane: Codeforge -- hard coding and agentic reasoning).
|
||||
|
||||
What it does, per query:
|
||||
1. generates a random schema (2-3 tables, INTEGER/TEXT/REAL columns, NULLs, 0-12 rows),
|
||||
2. generates one random SELECT in the supported grammar,
|
||||
3. creates the SAME schema in an in-memory sqlite3 database and executes the SAME SQL text
|
||||
in both engines,
|
||||
4. compares the result sets -- strictly, by type category (int / real / text / null) and by
|
||||
value -- as an ordered sequence when the query has ORDER BY, else as a multiset.
|
||||
|
||||
It exits 0 only when ALL of these hold, so the exit code cannot pass vacuously:
|
||||
* sqlite3 is importable and every generated statement ran on BOTH engines without error,
|
||||
* the requested number of queries actually executed,
|
||||
* every query shape was exercised (const / scan / where / join / group / agg / distinct),
|
||||
* at least a set fraction of queries returned at least one row, and the battery compared at
|
||||
least a set number of rows (so agreement is not agreement about empty sets),
|
||||
* zero mismatches,
|
||||
* with --mutants: every seeded engine defect is CAUGHT (each mutant must produce at least
|
||||
one mismatch). A mutant that survives means this battery has no teeth for that semantics.
|
||||
|
||||
Generator constraints (each one is there to keep the oracle deterministic, and each is a
|
||||
documented hole rather than an accident):
|
||||
* every ORDER BY is made total by appending the `id` columns of the FROM tables, so ties
|
||||
cannot reorder differently in the two engines;
|
||||
* DISTINCT queries are never ordered (SQLite requires ORDER BY terms to be in the result
|
||||
set, and ordered ties would not be comparable) -- they are compared as multisets;
|
||||
* LIMIT/OFFSET only appear with an ORDER BY;
|
||||
* with GROUP BY, the select list is group keys plus aggregates only -- never a bare column
|
||||
of a multi-row group, whose value SQLite leaves unspecified.
|
||||
|
||||
Usage:
|
||||
python3 diff_sqlite.py [--queries N] [--seed S] [--mutants] [--max-print K] [--quiet]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import random
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import minisql # noqa: E402
|
||||
|
||||
SHAPES = ("const", "scan", "where", "join", "group", "agg", "distinct")
|
||||
|
||||
TEXT_POOL = ["apple", "banana", "cherry", "a", "ab", "", "A", "10", "20"]
|
||||
LIKE_POOL = ["%a%", "a%", "%b", "_", "ab", "A%", "%", "", "c_erry", "ap_le", "%rry"]
|
||||
NUM_LITERALS = ["0", "1", "-1", "2", "5", "10", "20", "3.5", "5.5", "0.0"]
|
||||
STR_NUM_LITERALS = ["0", "5", "10", "20", "5.5", "abc", ""]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# schema
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def make_schema(rng):
|
||||
"""Return (spec, rows): spec is {table: [(col, declared_type)]}, rows is {table: [[v...]]}.
|
||||
|
||||
`id` is a unique non-NULL INTEGER in every table; it is the tiebreaker the generator
|
||||
appends to every ORDER BY, so ordered results are deterministic in both engines.
|
||||
"""
|
||||
spec = {
|
||||
"t1": [("id", "INTEGER"), ("a", "INTEGER"), ("b", "TEXT"), ("c", "REAL")],
|
||||
"t2": [("id", "INTEGER"), ("d", "INTEGER"), ("e", "TEXT")],
|
||||
}
|
||||
if rng.random() < 0.5:
|
||||
spec["t3"] = [("id", "INTEGER"), ("f", "REAL"), ("g", "TEXT")]
|
||||
|
||||
rows = {}
|
||||
for name, cols in spec.items():
|
||||
n = rng.randint(0, 12)
|
||||
table_rows = []
|
||||
for i in range(1, n + 1):
|
||||
row = []
|
||||
for cname, ctype in cols:
|
||||
if cname == "id":
|
||||
row.append(i)
|
||||
elif rng.random() < 0.2:
|
||||
row.append(None)
|
||||
elif ctype == "INTEGER":
|
||||
row.append(rng.randint(0, 20))
|
||||
elif ctype == "REAL":
|
||||
row.append(round(rng.uniform(0, 10), 1))
|
||||
else:
|
||||
row.append(rng.choice(TEXT_POOL))
|
||||
table_rows.append(row)
|
||||
rows[name] = table_rows
|
||||
return spec, rows
|
||||
|
||||
|
||||
def load_sqlite(con, spec, rows):
|
||||
for name, cols in spec.items():
|
||||
con.execute("CREATE TABLE %s (%s)" % (
|
||||
name, ", ".join("%s %s" % (c, t) for c, t in cols)))
|
||||
if rows[name]:
|
||||
con.executemany("INSERT INTO %s VALUES (%s)" % (
|
||||
name, ", ".join("?" * len(cols))), rows[name])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# query generator
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class Gen:
|
||||
"""Emits one random SELECT. All column references are drawn from the tables named in
|
||||
`scope`, which the query method sets to exactly the FROM tables before generating."""
|
||||
|
||||
def __init__(self, rng, spec, rows):
|
||||
self.rng = rng
|
||||
self.spec = spec
|
||||
self.rows = rows
|
||||
self.tables = list(spec)
|
||||
self.scope = []
|
||||
self.pool_num = [] # [(table, col)] with INTEGER/REAL type, inside scope
|
||||
self.pool_text = [] # [(table, col)] with TEXT type, inside scope
|
||||
self.pool_all = [] # [(table, col)] any type except id
|
||||
self.pool_id = [] # [(table, "id")]
|
||||
|
||||
# -- scope ------------------------------------------------------------
|
||||
def set_scope(self, tables):
|
||||
self.scope = list(tables)
|
||||
self.pool_num, self.pool_text, self.pool_all, self.pool_id = [], [], [], []
|
||||
for t in self.scope:
|
||||
for c, ty in self.spec[t]:
|
||||
if c == "id":
|
||||
self.pool_id.append((t, c))
|
||||
else:
|
||||
self.pool_all.append((t, c))
|
||||
(self.pool_num if ty in ("INTEGER", "REAL") else self.pool_text).append((t, c))
|
||||
|
||||
def ref(self, tc, qualify):
|
||||
t, c = tc
|
||||
return "%s.%s" % (t, c) if qualify else c
|
||||
|
||||
def pick(self, pool):
|
||||
return self.rng.choice(pool) if pool else None
|
||||
|
||||
# -- literals ---------------------------------------------------------
|
||||
def num_literal(self):
|
||||
return self.rng.choice(NUM_LITERALS)
|
||||
|
||||
def text_literal(self):
|
||||
return "'%s'" % self.rng.choice(TEXT_POOL)
|
||||
|
||||
# -- expressions ------------------------------------------------------
|
||||
def numeric_expr(self, qualify, depth=0):
|
||||
pool = self.pool_num
|
||||
r = self.rng.random()
|
||||
if not pool or r < 0.4 or depth >= 2:
|
||||
return self.ref(self.pick(pool), qualify) if pool else self.num_literal()
|
||||
if r < 0.55:
|
||||
return self.num_literal()
|
||||
if r < 0.68:
|
||||
return "(%s + %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal())
|
||||
if r < 0.8:
|
||||
return "(%s * %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal())
|
||||
if r < 0.9:
|
||||
return "(%s - %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal())
|
||||
# division is here on purpose: SQLite's `/` stays INTEGER for two integers, and the
|
||||
# `int-div-becomes-real` mutant exists to prove this battery notices when it does not.
|
||||
return "(%s / %s)" % (self.numeric_expr(qualify, depth + 1),
|
||||
self.rng.choice(["2", "3", "1", "4"]))
|
||||
|
||||
def scalar_expr(self, qualify, depth=0):
|
||||
r = self.rng.random()
|
||||
if r < 0.45:
|
||||
return self.numeric_expr(qualify, depth)
|
||||
if r < 0.6 and self.pool_text:
|
||||
return self.ref(self.pick(self.pool_text), qualify)
|
||||
if r < 0.7:
|
||||
return self.text_literal()
|
||||
if r < 0.8 and depth < 2:
|
||||
return "CASE WHEN %s THEN %s ELSE %s END" % (
|
||||
self.predicate(qualify, depth + 1), self.scalar_expr(qualify, depth + 1),
|
||||
self.scalar_expr(qualify, depth + 1))
|
||||
return self.predicate(qualify, depth + 1)
|
||||
|
||||
def predicate(self, qualify, depth=0):
|
||||
r = self.rng.random()
|
||||
if depth >= 2:
|
||||
r = min(r, 0.59)
|
||||
if r < 0.35:
|
||||
return self.comparison(qualify)
|
||||
if r < 0.45:
|
||||
return "(%s AND %s)" % (self.predicate(qualify, depth + 1),
|
||||
self.predicate(qualify, depth + 1))
|
||||
if r < 0.55:
|
||||
return "(%s OR %s)" % (self.predicate(qualify, depth + 1),
|
||||
self.predicate(qualify, depth + 1))
|
||||
if r < 0.6:
|
||||
return "NOT %s" % self.predicate(qualify, depth + 1)
|
||||
if r < 0.68 and self.pool_all:
|
||||
tc = self.pick(self.pool_all)
|
||||
return "%s IS%s NULL" % (self.ref(tc, qualify),
|
||||
"" if self.rng.random() < 0.5 else " NOT")
|
||||
if r < 0.82 and self.pool_num:
|
||||
items = ", ".join(self.num_literal() for _ in range(self.rng.randint(1, 3)))
|
||||
return "%s%s IN (%s)" % (self.ref(self.pick(self.pool_num), qualify),
|
||||
" NOT" if self.rng.random() < 0.3 else "", items)
|
||||
if r < 0.88 and self.pool_text:
|
||||
items = ", ".join(self.text_literal() for _ in range(self.rng.randint(1, 3)))
|
||||
return "%s%s IN (%s)" % (self.ref(self.pick(self.pool_text), qualify),
|
||||
" NOT" if self.rng.random() < 0.3 else "", items)
|
||||
if r < 0.93 and self.pool_num:
|
||||
return "%s%s BETWEEN %s AND %s" % (
|
||||
self.ref(self.pick(self.pool_num), qualify),
|
||||
" NOT" if self.rng.random() < 0.3 else "",
|
||||
self.num_literal(), self.num_literal())
|
||||
if self.pool_text:
|
||||
return "%s%s LIKE '%s'" % (self.ref(self.pick(self.pool_text), qualify),
|
||||
" NOT" if self.rng.random() < 0.3 else "",
|
||||
self.rng.choice(LIKE_POOL))
|
||||
return self.comparison(qualify)
|
||||
|
||||
def comparison(self, qualify):
|
||||
"""The mixed-affinity forms are deliberate: they are where an engine that ignores
|
||||
SQLite's two affinity rules diverges from it."""
|
||||
r = self.rng.random()
|
||||
op = self.rng.choice(["=", "<>", "<", "<=", ">", ">="])
|
||||
if r < 0.35 and self.pool_num:
|
||||
return "%s %s %s" % (self.ref(self.pick(self.pool_num), qualify), op,
|
||||
self.num_literal())
|
||||
if r < 0.55 and self.pool_text:
|
||||
return "%s %s %s" % (self.ref(self.pick(self.pool_text), qualify), op,
|
||||
self.text_literal())
|
||||
if r < 0.7 and self.pool_num:
|
||||
# numeric affinity is applied to the text literal
|
||||
return "%s %s '%s'" % (self.ref(self.pick(self.pool_num), qualify), op,
|
||||
self.rng.choice(STR_NUM_LITERALS))
|
||||
if r < 0.85 and self.pool_text:
|
||||
# TEXT affinity is applied to the numeric literal
|
||||
return "%s %s %s" % (self.ref(self.pick(self.pool_text), qualify), op,
|
||||
self.rng.choice(["0", "5", "10", "20", "-1"]))
|
||||
if self.pool_num:
|
||||
return "%s %s %s" % (self.numeric_expr(qualify, 1), op, self.num_literal())
|
||||
return self.comparison(qualify) if (self.pool_text or self.pool_num) else "1 = 1"
|
||||
|
||||
# -- select items -----------------------------------------------------
|
||||
def agg_item(self, qualify, i):
|
||||
fn = self.rng.choice(["COUNT", "SUM", "AVG", "MIN", "MAX"])
|
||||
if fn == "COUNT" and self.rng.random() < 0.4:
|
||||
return "COUNT(*) AS agg%d" % i
|
||||
pool = self.pool_num if (fn in ("SUM", "AVG") and self.pool_num) else self.pool_all
|
||||
if not pool:
|
||||
return "COUNT(*) AS agg%d" % i
|
||||
return "%s(%s) AS agg%d" % (fn, self.ref(self.pick(pool), qualify), i)
|
||||
|
||||
def items(self, qualify, agg=False, group_keys=None):
|
||||
out = []
|
||||
if not agg and self.rng.random() < 0.25:
|
||||
out.append("*")
|
||||
for i in range(self.rng.randint(1, 3)):
|
||||
if agg:
|
||||
if group_keys and self.rng.random() < 0.35:
|
||||
out.append("%s AS gk%d" % (self.rng.choice(group_keys), i))
|
||||
else:
|
||||
out.append(self.agg_item(qualify, i))
|
||||
else:
|
||||
e = self.scalar_expr(qualify)
|
||||
out.append("%s AS x%d" % (e, i) if self.rng.random() < 0.6 else e)
|
||||
if not out:
|
||||
out.append("*")
|
||||
return out
|
||||
|
||||
def having(self, gkeys):
|
||||
r = self.rng.random()
|
||||
if r < 0.45:
|
||||
return "COUNT(*) > %d" % self.rng.randint(0, 3)
|
||||
if r < 0.6 and self.pool_num:
|
||||
return "SUM(%s) > %s" % (self.ref(self.pick(self.pool_num), False),
|
||||
self.rng.choice(["5", "10", "20", "0"]))
|
||||
if r < 0.7 and self.pool_num:
|
||||
return "AVG(%s) > %s" % (self.ref(self.pick(self.pool_num), False),
|
||||
self.rng.choice(["1", "5", "10"]))
|
||||
if r < 0.8 and self.pool_all:
|
||||
return "MAX(%s) > %s" % (self.ref(self.pick(self.pool_all), False),
|
||||
self.rng.choice(["5", "10", "1.5"]))
|
||||
if r < 0.9 and gkeys:
|
||||
return "%s IS NOT NULL" % self.rng.choice(gkeys)
|
||||
return "COUNT(*) > 1"
|
||||
|
||||
def order_key(self, qualify):
|
||||
"""ORDER BY terms are always column references: a bare integer literal in ORDER BY is
|
||||
an ORDINAL in SQLite, not a constant, and the oracle rejects it out of range."""
|
||||
pool = list(self.pool_all) + list(self.pool_id)
|
||||
if not pool:
|
||||
return "1 = 1"
|
||||
return self.ref(self.rng.choice(pool), qualify)
|
||||
|
||||
def order_tail(self, keys, tiebreak):
|
||||
"""keys: ordering exprs; tiebreak: exprs that make the order total."""
|
||||
terms = ["%s%s" % (k, " DESC" if self.rng.random() < 0.4 else "") for k in keys]
|
||||
terms.extend(tiebreak)
|
||||
tail = " ORDER BY " + ", ".join(terms)
|
||||
if self.rng.random() < 0.35:
|
||||
tail += " LIMIT %d" % self.rng.randint(0, 6)
|
||||
if self.rng.random() < 0.4:
|
||||
tail += " OFFSET %d" % self.rng.randint(0, 4)
|
||||
return tail
|
||||
|
||||
# -- whole queries ----------------------------------------------------
|
||||
def query(self, shape):
|
||||
rng = self.rng
|
||||
t = rng.choice(self.tables)
|
||||
|
||||
if shape == "const":
|
||||
self.set_scope([])
|
||||
return "SELECT " + ", ".join(self.scalar_expr(False) for _ in range(rng.randint(1, 3)))
|
||||
|
||||
if shape in ("scan", "where", "distinct"):
|
||||
self.set_scope([t])
|
||||
sql = "SELECT " + ("DISTINCT " if shape == "distinct" else "") \
|
||||
+ ", ".join(self.items(False))
|
||||
sql += " FROM " + t
|
||||
if shape in ("where", "distinct") or rng.random() < 0.4:
|
||||
sql += " WHERE " + self.predicate(False)
|
||||
if shape != "distinct":
|
||||
keys = [self.order_key(False) for _ in range(rng.randint(1, 2))]
|
||||
sql += self.order_tail(keys, ["%s.id" % t])
|
||||
return sql
|
||||
|
||||
if shape == "join":
|
||||
others = [x for x in self.tables if x != t]
|
||||
if not others:
|
||||
return self.query("scan")
|
||||
u = rng.choice(others)
|
||||
self.set_scope([t, u])
|
||||
kind = "LEFT JOIN" if rng.random() < 0.45 else "JOIN"
|
||||
cands = [c for (tt, c) in self.pool_num if tt == t] or ["id"]
|
||||
on = rng.choice([
|
||||
"%s.id = %s.id" % (t, u),
|
||||
"%s.id = %s.id AND %s" % (t, u, self.predicate(True)),
|
||||
"%s.%s = %s.id" % (t, rng.choice(cands), u),
|
||||
])
|
||||
grouped = rng.random() < 0.5
|
||||
if grouped:
|
||||
gkeys = ["%s.id" % t, "%s.id" % u]
|
||||
sql = "SELECT " + ", ".join(self.items(True, agg=True, group_keys=gkeys))
|
||||
sql += " FROM %s %s %s ON %s" % (t, kind, u, on)
|
||||
if rng.random() < 0.5:
|
||||
sql += " WHERE " + self.predicate(True)
|
||||
sql += " GROUP BY " + ", ".join(gkeys)
|
||||
sql += self.order_tail(["COUNT(*)"], gkeys)
|
||||
else:
|
||||
sql = "SELECT " + ", ".join(self.items(True, agg=False))
|
||||
sql += " FROM %s %s %s ON %s" % (t, kind, u, on)
|
||||
if rng.random() < 0.5:
|
||||
sql += " WHERE " + self.predicate(True)
|
||||
sql += self.order_tail(["%s.id" % t], ["%s.id" % t, "%s.id" % u])
|
||||
return sql
|
||||
|
||||
if shape == "agg":
|
||||
self.set_scope([t])
|
||||
sql = "SELECT " + ", ".join(self.items(False, agg=True))
|
||||
sql += " FROM " + t
|
||||
if rng.random() < 0.6:
|
||||
sql += " WHERE " + self.predicate(False)
|
||||
if rng.random() < 0.4:
|
||||
sql += " HAVING " + self.having(None)
|
||||
return sql
|
||||
|
||||
# group
|
||||
self.set_scope([t])
|
||||
gkeys = []
|
||||
for _ in range(rng.randint(1, 2)):
|
||||
if self.pool_all:
|
||||
gkeys.append(self.ref(self.pick(self.pool_all), False))
|
||||
if not gkeys:
|
||||
gkeys = ["%s.id" % t]
|
||||
sql = "SELECT " + ", ".join(self.items(False, agg=True, group_keys=gkeys))
|
||||
sql += " FROM " + t
|
||||
if rng.random() < 0.6:
|
||||
sql += " WHERE " + self.predicate(False)
|
||||
sql += " GROUP BY " + ", ".join(gkeys)
|
||||
if rng.random() < 0.5:
|
||||
sql += " HAVING " + self.having(gkeys)
|
||||
sql += self.order_tail(["COUNT(*)"], gkeys)
|
||||
return sql
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# comparison
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def cat(v):
|
||||
if v is None:
|
||||
return "null"
|
||||
if isinstance(v, bool):
|
||||
return "int"
|
||||
if isinstance(v, int):
|
||||
return "int"
|
||||
if isinstance(v, float):
|
||||
return "real"
|
||||
if isinstance(v, str):
|
||||
return "text"
|
||||
return "other:" + type(v).__name__
|
||||
|
||||
|
||||
def same_value(a, b):
|
||||
return cat(a) == cat(b) and a == b
|
||||
|
||||
|
||||
def canon_row(row):
|
||||
out = []
|
||||
for v in row:
|
||||
c = cat(v)
|
||||
out.append(("num", float(v)) if c in ("int", "real") else (c, str(v)))
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def same_row(ra, rb):
|
||||
return len(ra) == len(rb) and all(same_value(a, b) for a, b in zip(ra, rb))
|
||||
|
||||
|
||||
def compare_rows(got, want, ordered):
|
||||
"""None when the two result sets agree, else a short description of the first difference."""
|
||||
if len(got) != len(want):
|
||||
return "row count %d != %d" % (len(got), len(want))
|
||||
if ordered:
|
||||
for i, (g, w) in enumerate(zip(got, want)):
|
||||
if not same_row(g, w):
|
||||
return "row %d: %r != %r" % (i, g, w)
|
||||
return None
|
||||
for g, w in zip(sorted(got, key=canon_row), sorted(want, key=canon_row)):
|
||||
if not same_row(g, w):
|
||||
return "multiset row %r != %r" % (g, w)
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# runner
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def run_battery(queries, seed, max_print=5, quiet=False):
|
||||
rng = random.Random(seed)
|
||||
con = sqlite3.connect(":memory:")
|
||||
shape_counts = dict.fromkeys(SHAPES, 0)
|
||||
mismatches, errors = [], []
|
||||
rows_compared = nonempty = ordered_count = 0
|
||||
|
||||
for i in range(queries):
|
||||
shape = SHAPES[i % len(SHAPES)]
|
||||
spec, rows = make_schema(rng)
|
||||
con.close()
|
||||
con = sqlite3.connect(":memory:")
|
||||
load_sqlite(con, spec, rows)
|
||||
mtables = {name: minisql.table_from(cols, rows[name]) for name, cols in spec.items()}
|
||||
sql = Gen(rng, spec, rows).query(shape)
|
||||
shape_counts[shape] += 1
|
||||
ordered = " ORDER BY " in sql
|
||||
ordered_count += 1 if ordered else 0
|
||||
try:
|
||||
want = [list(r) for r in con.execute(sql).fetchall()]
|
||||
except sqlite3.Error as exc:
|
||||
errors.append((sql, str(exc)))
|
||||
continue
|
||||
try:
|
||||
_names, got = minisql.execute(sql, mtables)
|
||||
except Exception as exc: # noqa: BLE001 - any engine failure is red
|
||||
mismatches.append((shape, sql, "minisql raised %s: %s" % (type(exc).__name__, exc),
|
||||
None, want))
|
||||
continue
|
||||
rows_compared += len(want)
|
||||
nonempty += 1 if want else 0
|
||||
diff = compare_rows(got, want, ordered)
|
||||
if diff is not None:
|
||||
mismatches.append((shape, sql, diff, got, want))
|
||||
con.close()
|
||||
|
||||
report = {
|
||||
"queries": queries, "seed": seed, "shape_counts": shape_counts,
|
||||
"rows_compared": rows_compared, "nonempty": nonempty, "ordered": ordered_count,
|
||||
"mismatches": mismatches, "errors": errors,
|
||||
}
|
||||
if not quiet:
|
||||
print("minisql differential battery vs sqlite3 %s" % sqlite3.sqlite_version)
|
||||
print("seed=%d queries=%d" % (seed, queries))
|
||||
print("shape counts: " + " ".join("%s=%d" % (k, shape_counts[k]) for k in SHAPES))
|
||||
print("ordered queries: %d rows compared: %d queries returning >=1 row: %d"
|
||||
% (ordered_count, rows_compared, nonempty))
|
||||
for shape, sql, diff, got, want in mismatches[:max_print]:
|
||||
print("\nMISMATCH [%s] %s\n %s" % (shape, diff, sql))
|
||||
print(" oracle: %r" % (want[:6],))
|
||||
print(" engine: %r" % (got[:6] if got is not None else None,))
|
||||
if len(mismatches) > max_print:
|
||||
print("... %d more mismatches" % (len(mismatches) - max_print))
|
||||
for sql, exc in errors[:max_print]:
|
||||
print("\nHARNESS ERROR (oracle refused generated SQL): %s\n %s" % (exc, sql))
|
||||
if len(errors) > max_print:
|
||||
print("... %d more harness errors" % (len(errors) - max_print))
|
||||
return report
|
||||
|
||||
|
||||
def check_report(report):
|
||||
"""The non-vacuity guards. Returns a list of failure strings."""
|
||||
bad = []
|
||||
if report["errors"]:
|
||||
bad.append("%d generated statements were refused by the oracle (harness defect)"
|
||||
% len(report["errors"]))
|
||||
if report["mismatches"]:
|
||||
bad.append("%d mismatches" % len(report["mismatches"]))
|
||||
if report["queries"] < len(SHAPES):
|
||||
bad.append("too few queries to exercise every shape")
|
||||
for shape in SHAPES:
|
||||
if report["shape_counts"][shape] == 0:
|
||||
bad.append("shape %s was never exercised" % shape)
|
||||
if report["nonempty"] < 0.5 * report["queries"]:
|
||||
bad.append("only %d/%d queries returned rows -- agreement would be too close to vacuous"
|
||||
% (report["nonempty"], report["queries"]))
|
||||
if report["rows_compared"] < report["queries"]:
|
||||
bad.append("only %d rows compared for %d queries"
|
||||
% (report["rows_compared"], report["queries"]))
|
||||
if report["ordered"] < 0.2 * report["queries"]:
|
||||
bad.append("only %d ordered queries -- ORDER BY is barely covered" % report["ordered"])
|
||||
return bad
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# mutation control: prove the battery can go RED
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _mutants():
|
||||
orig = {
|
||||
"compare_values": minisql.compare_values,
|
||||
"join_relation": minisql.join_relation,
|
||||
"sort_key": minisql.sort_key,
|
||||
"agg_sum": minisql.agg_sum,
|
||||
"agg_count": minisql.agg_count,
|
||||
"arith_div": minisql.arith_div,
|
||||
"_tri_and": minisql._tri_and,
|
||||
"like_match": minisql.like_match,
|
||||
}
|
||||
|
||||
def m_null_eq(a, b, aff_a=None, aff_b=None):
|
||||
if a is None or b is None:
|
||||
return 0 # NULL = NULL is TRUE, and NULL = 1 is TRUE
|
||||
return orig["compare_values"](a, b, aff_a, aff_b)
|
||||
|
||||
def m_left_inner(left, right, kind, on_eval):
|
||||
return orig["join_relation"](left, right, "inner", on_eval)
|
||||
|
||||
def m_nulls_last(v):
|
||||
if v is None:
|
||||
return (3, 0) # NULLs sort last on ASC
|
||||
return (min(minisql.class_rank(v), 2), v)
|
||||
|
||||
def m_sum_empty(values):
|
||||
r = orig["agg_sum"](values)
|
||||
return 0 if r is None else r
|
||||
|
||||
def m_count_nulls(values, n_rows):
|
||||
return n_rows # COUNT(x) counts NULLs too
|
||||
|
||||
def m_div_float(a, b):
|
||||
r = orig["arith_div"](a, b)
|
||||
return None if r is None else float(r)
|
||||
|
||||
def m_and_absorb(a, b):
|
||||
ta, tb = minisql.tri(a), minisql.tri(b)
|
||||
if ta is None or tb is None:
|
||||
return None # NULL absorbs FALSE
|
||||
return 0 if (ta == 0 or tb == 0) else 1
|
||||
|
||||
def m_like_case_sensitive(value, pattern):
|
||||
if value is None or pattern is None:
|
||||
return None
|
||||
import re as _re
|
||||
rx = _re.compile("(?s)\\A" + "".join(
|
||||
".*" if ch == "%" else ("." if ch == "_" else _re.escape(ch))
|
||||
for ch in pattern) + "\\Z")
|
||||
return 1 if rx.match(str(value)) else 0
|
||||
|
||||
return [
|
||||
("null-eq", "compare_values", m_null_eq),
|
||||
("left-join-as-inner", "join_relation", m_left_inner),
|
||||
("nulls-last-asc", "sort_key", m_nulls_last),
|
||||
("sum-empty-zero", "agg_sum", m_sum_empty),
|
||||
("count-counts-nulls", "agg_count", m_count_nulls),
|
||||
("int-div-becomes-real", "arith_div", m_div_float),
|
||||
("and-null-absorbs-false", "_tri_and", m_and_absorb),
|
||||
("like-case-sensitive", "like_match", m_like_case_sensitive),
|
||||
]
|
||||
|
||||
|
||||
def run_mutants(queries, seed, quiet=False):
|
||||
results = []
|
||||
for name, attr, fn in _mutants():
|
||||
original = getattr(minisql, attr)
|
||||
setattr(minisql, attr, fn)
|
||||
try:
|
||||
rep = run_battery(queries, seed, max_print=0, quiet=True)
|
||||
finally:
|
||||
setattr(minisql, attr, original)
|
||||
caught = len(rep["mismatches"])
|
||||
results.append((name, attr, caught))
|
||||
if not quiet:
|
||||
print(" mutant %-24s patches %-15s -> %s"
|
||||
% (name, attr, "CAUGHT (%d mismatches)" % caught if caught
|
||||
else "SURVIVED -- battery has no teeth here"))
|
||||
return results
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description="minisql vs sqlite3 differential battery")
|
||||
ap.add_argument("--queries", type=int, default=1400)
|
||||
ap.add_argument("--seed", type=int, default=20261022)
|
||||
ap.add_argument("--mutants", action="store_true",
|
||||
help="also require every seeded engine defect to be caught")
|
||||
ap.add_argument("--mutant-queries", type=int, default=140)
|
||||
ap.add_argument("--max-print", type=int, default=5)
|
||||
ap.add_argument("--quiet", action="store_true")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
rep = run_battery(args.queries, args.seed, max_print=args.max_print, quiet=args.quiet)
|
||||
bad = check_report(rep)
|
||||
|
||||
mutants_caught = 0
|
||||
if args.mutants:
|
||||
if not args.quiet:
|
||||
print("\nmutation control (every seeded defect must be caught):")
|
||||
results = run_mutants(args.mutant_queries, args.seed, quiet=args.quiet)
|
||||
survived = [n for n, _a, c in results if not c]
|
||||
mutants_caught = len(results) - len(survived)
|
||||
if not args.quiet:
|
||||
print("mutants caught: %d/%d" % (mutants_caught, len(results)))
|
||||
if survived:
|
||||
bad.append("mutants survived (battery has no teeth): " + ", ".join(survived))
|
||||
|
||||
if not args.quiet:
|
||||
print()
|
||||
for b in bad:
|
||||
print("FAIL: " + b)
|
||||
if not bad:
|
||||
print("all %d queries agreed with sqlite3; %d rows compared; %d mutants caught"
|
||||
% (rep["queries"], rep["rows_compared"], mutants_caught))
|
||||
print("RESULT: " + ("FAIL" if bad else "PASS"))
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,46 @@
|
||||
# raglite — inverted index + BM25 retrieval (stdlib only)
|
||||
|
||||
Minimal passage retrieval for agents: index a file or directory of markdown/text,
|
||||
rank with BM25, get JSON hits. No dependencies, no network, deterministic.
|
||||
|
||||
## Files
|
||||
|
||||
- `raglite.py` — library + CLI
|
||||
- `test_raglite.py` — self-contained test suite (own fixtures, own temp dirs)
|
||||
|
||||
## Library
|
||||
|
||||
```python
|
||||
import raglite
|
||||
idx = raglite.build_index(["./docs"]) # file or directory; recursive
|
||||
raglite.save_index(idx, "index.json")
|
||||
idx = raglite.load_index("index.json")
|
||||
hits = raglite.search(idx, "bm25 ranking", k=3)
|
||||
# -> [{"id": "docs/bm25.md::0", "score": 3.63..., "text": "..."}]
|
||||
```
|
||||
|
||||
A passage is a blank-line-delimited block, addressed `<path>::<paragraph-index>`.
|
||||
The index is plain JSON (`postings`, `lengths`, `texts`, `n_docs`, `avgdl`, `k1`, `b`).
|
||||
|
||||
## CLI
|
||||
|
||||
```
|
||||
python3 raglite.py index ./docs --out index.json
|
||||
python3 raglite.py query index.json "bm25 ranking" -k 3
|
||||
python3 raglite.py query index.json "bm25 ranking" -k 3 --json
|
||||
```
|
||||
|
||||
## Evidence (measured, 2026-10-22)
|
||||
|
||||
`python3 test_raglite.py` → **25/25 checks pass, exit 0**.
|
||||
|
||||
Checks cover: tokenizer punctuation/lowercasing, passage splitting, passage count
|
||||
(4 docs × 2 paragraphs = 8), top-hit correctness for three queries, unknown-term →
|
||||
no hits, positive and descending scores, a negative control (codeforge must not rank
|
||||
for a kibana query), JSON round-trip identity, exact BM25 constants on a 2-doc corpus
|
||||
(compared to values recomputed independently in `bm25_probe.py`), tf ordering, and the
|
||||
CLI end-to-end via subprocess (exit codes, passage count, valid JSON, top hit).
|
||||
|
||||
Real-corpus smoke test: `raglite.py index /root/troupe-standup/wizard-work/null`
|
||||
→ **139 passages**; query `"json patch apply"` → top hit `jsonpatch/README.md::0`;
|
||||
query `"scheduler cron"` → top hit `schedkit/README.md::0`.
|
||||
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""raglite -- minimal retrieval for agents: inverted index + BM25 over passages.
|
||||
|
||||
Stdlib only. Deterministic. No network. No dependencies.
|
||||
|
||||
A "passage" is a blank-line-delimited block of a text/markdown file, addressed
|
||||
as "<path>::<paragraph-index>". The index is plain JSON so it can be diffed,
|
||||
cached and inspected by hand.
|
||||
|
||||
Library:
|
||||
idx = raglite.build_index(["./docs"]) # path may be a file or a directory
|
||||
raglite.save_index(idx, "index.json")
|
||||
idx = raglite.load_index("index.json")
|
||||
hits = raglite.search(idx, "bm25 ranking", k=3)
|
||||
|
||||
CLI:
|
||||
python3 raglite.py index ./docs --out index.json
|
||||
python3 raglite.py query index.json "bm25 ranking" -k 3 --json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
TOKEN_RE = re.compile(r"[A-Za-z0-9_']+")
|
||||
PASSAGE_SPLIT = re.compile(r"\n[ \t]*\n")
|
||||
DEFAULT_EXTENSIONS = (".md", ".txt")
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
"""Lowercased alphanumeric tokens; punctuation is a separator."""
|
||||
return TOKEN_RE.findall(text.lower())
|
||||
|
||||
|
||||
def split_passages(text: str) -> list[str]:
|
||||
"""Split on blank lines, drop empties, keep original order."""
|
||||
return [p.strip() for p in PASSAGE_SPLIT.split(text) if p.strip()]
|
||||
|
||||
|
||||
def build_index(paths, k1: float = 1.5, b: float = 0.75,
|
||||
extensions=DEFAULT_EXTENSIONS) -> dict:
|
||||
"""Build a BM25 index over every passage of every file under `paths`."""
|
||||
postings: dict[str, Counter] = defaultdict(Counter)
|
||||
lengths: dict[str, int] = {}
|
||||
texts: dict[str, str] = {}
|
||||
n = 0
|
||||
|
||||
for raw in paths:
|
||||
p = Path(raw)
|
||||
if p.is_dir():
|
||||
files = sorted(q for q in p.rglob("*")
|
||||
if q.suffix.lower() in extensions)
|
||||
elif p.is_file():
|
||||
files = [p]
|
||||
else:
|
||||
raise FileNotFoundError(f"no such path: {raw}")
|
||||
|
||||
for f in files:
|
||||
body = f.read_text(encoding="utf-8", errors="replace")
|
||||
for i, passage in enumerate(split_passages(body)):
|
||||
toks = tokenize(passage)
|
||||
if not toks:
|
||||
continue
|
||||
pid = f"{f}::{i}"
|
||||
for term, tf in Counter(toks).items():
|
||||
postings[term][pid] = tf
|
||||
lengths[pid] = len(toks)
|
||||
texts[pid] = passage
|
||||
n += 1
|
||||
|
||||
total = sum(lengths.values())
|
||||
return {
|
||||
"k1": k1,
|
||||
"b": b,
|
||||
"n_docs": n,
|
||||
"avgdl": (total / n) if n else 0.0,
|
||||
"lengths": lengths,
|
||||
"texts": texts,
|
||||
"postings": {t: dict(c) for t, c in postings.items()},
|
||||
}
|
||||
|
||||
|
||||
def _idf(df: int, n: int) -> float:
|
||||
"""BM25 idf, Robertson's non-negative form."""
|
||||
return math.log(1.0 + (n - df + 0.5) / (df + 0.5))
|
||||
|
||||
|
||||
def search(idx: dict, query: str, k: int = 5) -> list[dict]:
|
||||
"""Rank passages for `query`; returns <= k hits, best first.
|
||||
|
||||
Each hit: {"id", "score", "text"}. Score is the raw BM25 sum (unrounded).
|
||||
"""
|
||||
k1, b = idx["k1"], idx["b"]
|
||||
n, avgdl = idx["n_docs"], idx["avgdl"]
|
||||
if not avgdl or k <= 0:
|
||||
return []
|
||||
|
||||
scores: dict[str, float] = defaultdict(float)
|
||||
for term in set(tokenize(query)):
|
||||
post = idx["postings"].get(term)
|
||||
if not post:
|
||||
continue
|
||||
idf = _idf(len(post), n)
|
||||
for pid, tf in post.items():
|
||||
dl = idx["lengths"][pid]
|
||||
denom = tf + k1 * (1.0 - b + b * dl / avgdl)
|
||||
scores[pid] += idf * (tf * (k1 + 1.0)) / denom
|
||||
|
||||
ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0]))
|
||||
return [{"id": pid, "score": s, "text": idx["texts"][pid]}
|
||||
for pid, s in ranked[:k]]
|
||||
|
||||
|
||||
def save_index(idx: dict, path) -> None:
|
||||
Path(path).write_text(json.dumps(idx, sort_keys=True), encoding="utf-8")
|
||||
|
||||
|
||||
def load_index(path) -> dict:
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(prog="raglite", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
pi = sub.add_parser("index", help="build an index over a file or directory")
|
||||
pi.add_argument("path")
|
||||
pi.add_argument("--out", required=True)
|
||||
pi.add_argument("--k1", type=float, default=1.5)
|
||||
pi.add_argument("--b", type=float, default=0.75)
|
||||
|
||||
pq = sub.add_parser("query", help="rank passages in an index")
|
||||
pq.add_argument("index")
|
||||
pq.add_argument("query")
|
||||
pq.add_argument("-k", type=int, default=5)
|
||||
pq.add_argument("--json", action="store_true", help="emit JSON hits")
|
||||
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
if args.cmd == "index":
|
||||
idx = build_index([args.path], k1=args.k1, b=args.b)
|
||||
save_index(idx, args.out)
|
||||
print(f"indexed {idx['n_docs']} passages from {args.path} -> {args.out}")
|
||||
return 0
|
||||
|
||||
idx = load_index(args.index)
|
||||
hits = search(idx, args.query, k=args.k)
|
||||
if args.json:
|
||||
print(json.dumps(hits, indent=2))
|
||||
else:
|
||||
for i, h in enumerate(hits, 1):
|
||||
print(f"{i}. [{h['score']:.6f}] {h['id']}")
|
||||
print(f" {h['text'][:160]}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Self-contained tests for raglite.py.
|
||||
|
||||
Creates its own fixtures in temp dirs, exercises the library and the CLI,
|
||||
and exits 0 only if every check passes. No network, no external deps.
|
||||
|
||||
Run: python3 /root/.hermes/wizard-outputs/null/test_raglite.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
import raglite # noqa: E402
|
||||
|
||||
FAILURES = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
if cond:
|
||||
print(f" ok {name}")
|
||||
else:
|
||||
print(f" FAIL {name} {detail}")
|
||||
FAILURES.append(name)
|
||||
|
||||
|
||||
def doc_of(pid: str) -> str:
|
||||
"""'path/to/codeforge.md::3' -> 'codeforge.md'"""
|
||||
return pid.split("::")[0].rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
DOCS = {
|
||||
"codeforge.md": (
|
||||
"The wizard forged the spell in the codeforge.\n"
|
||||
"\n"
|
||||
"The codeforge hums with runes and ember-light.\n"
|
||||
),
|
||||
"kibana.md": (
|
||||
"Kibana dashboards visualize logs from the elasticsearch cluster.\n"
|
||||
"\n"
|
||||
"Operators drill into slow queries here.\n"
|
||||
),
|
||||
"bm25.md": (
|
||||
"BM25 ranks passages by term frequency and inverse document frequency.\n"
|
||||
"\n"
|
||||
"It is a bag-of-words retrieval function.\n"
|
||||
),
|
||||
"standup.md": (
|
||||
"The troupe stands up at dawn.\n"
|
||||
"\n"
|
||||
"Each wizard reports their lane and blockers.\n"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def write_corpus(root):
|
||||
for name, body in DOCS.items():
|
||||
(root / name).write_text(body, encoding="utf-8")
|
||||
|
||||
|
||||
def test_tokenize():
|
||||
print("[tokenize]")
|
||||
toks = raglite.tokenize("Hello, WORLD! It's a BM25-test.")
|
||||
check("punctuation stripped, lowercased",
|
||||
toks == ["hello", "world", "it's", "a", "bm25", "test"], repr(toks))
|
||||
check("empty input -> empty list", raglite.tokenize("") == [])
|
||||
|
||||
|
||||
def test_split_passages():
|
||||
print("[split_passages]")
|
||||
parts = raglite.split_passages("a\n\nb\n\n\nc\n")
|
||||
check("blank-line split", parts == ["a", "b", "c"], repr(parts))
|
||||
check("whitespace-only block dropped", raglite.split_passages("a\n \nb") == ["a", "b"])
|
||||
|
||||
|
||||
def test_index_and_ranking(tmp):
|
||||
print("[index + ranking]")
|
||||
idx = raglite.build_index([str(tmp)])
|
||||
check("4 docs x 2 paragraphs -> 8 passages", idx["n_docs"] == 8, idx["n_docs"])
|
||||
check("avgdl is positive", idx["avgdl"] > 0, idx["avgdl"])
|
||||
|
||||
h = raglite.search(idx, "codeforge", k=3)
|
||||
check("query 'codeforge' returns hits", len(h) > 0)
|
||||
check("top hit is the codeforge passage",
|
||||
bool(h) and doc_of(h[0]["id"]) == "codeforge.md"
|
||||
and "codeforge" in h[0]["text"],
|
||||
h[0]["id"] if h else "none")
|
||||
|
||||
h = raglite.search(idx, "elasticsearch logs", k=3)
|
||||
check("top hit for 'elasticsearch logs' is kibana",
|
||||
bool(h) and doc_of(h[0]["id"]) == "kibana.md", h[0]["id"] if h else "none")
|
||||
|
||||
h = raglite.search(idx, "bm25 ranking", k=3)
|
||||
check("top hit for 'bm25 ranking' is bm25.md",
|
||||
bool(h) and doc_of(h[0]["id"]) == "bm25.md", h[0]["id"] if h else "none")
|
||||
|
||||
h = raglite.search(idx, "nonexistentterm", k=5)
|
||||
check("unknown term -> no hits", h == [], repr(h))
|
||||
|
||||
h = raglite.search(idx, "the", k=5)
|
||||
check("scores strictly positive", all(x["score"] > 0 for x in h), repr(h))
|
||||
check("scores sorted descending",
|
||||
all(h[i]["score"] >= h[i + 1]["score"] for i in range(len(h) - 1)), repr(h))
|
||||
|
||||
# Negative control: a kibana query must not surface the codeforge doc.
|
||||
h = raglite.search(idx, "kibana dashboards", k=5)
|
||||
check("negative control: codeforge not ranked for kibana query",
|
||||
all(doc_of(x["id"]) != "codeforge.md" for x in h),
|
||||
repr([x["id"] for x in h]))
|
||||
|
||||
|
||||
def test_roundtrip(tmp):
|
||||
print("[json round-trip]")
|
||||
idx = raglite.build_index([str(tmp)])
|
||||
path = tmp / "index.json"
|
||||
raglite.save_index(idx, path)
|
||||
check("index file written", path.exists() and path.stat().st_size > 0)
|
||||
|
||||
loaded = raglite.load_index(path)
|
||||
a = raglite.search(idx, "codeforge", k=5)
|
||||
b = raglite.search(loaded, "codeforge", k=5)
|
||||
check("loaded index gives identical hits",
|
||||
[(x["id"], x["score"]) for x in a] == [(x["id"], x["score"]) for x in b],
|
||||
f"{a} != {b}")
|
||||
|
||||
|
||||
def test_exact_bm25():
|
||||
"""Compare against BM25 values computed independently (bm25_probe.py)."""
|
||||
print("[exact bm25 constants]")
|
||||
with tempfile.TemporaryDirectory(prefix="raglite_tiny_") as d:
|
||||
tiny = Path(d)
|
||||
(tiny / "d1.md").write_text("a a b\n", encoding="utf-8")
|
||||
(tiny / "d2.md").write_text("a c\n", encoding="utf-8")
|
||||
idx = raglite.build_index([str(tiny)])
|
||||
|
||||
# Independently recomputed with the same Robertson formula (bm25_probe.py).
|
||||
EXPECTED = {
|
||||
("d1.md", "a"): 0.24472692187108,
|
||||
("d2.md", "a"): 0.20035335911423582,
|
||||
("d1.md", "b"): 0.6359148445504086,
|
||||
("d2.md", "c"): 0.7617001984175223,
|
||||
}
|
||||
for (doc, term), want in EXPECTED.items():
|
||||
hits = raglite.search(idx, term, k=10)
|
||||
got = next((x["score"] for x in hits if doc_of(x["id"]) == doc), None)
|
||||
check(f"score[{doc},{term}] == {want:.12f}",
|
||||
got is not None and abs(got - want) < 1e-12,
|
||||
f"got {got!r}")
|
||||
|
||||
# Higher term frequency must win for the same term.
|
||||
hits = raglite.search(idx, "a", k=2)
|
||||
check("tf ordering: d1 (tf=2) ranks above d2 (tf=1)",
|
||||
len(hits) == 2 and doc_of(hits[0]["id"]) == "d1.md", repr(hits))
|
||||
|
||||
|
||||
def test_cli(tmp):
|
||||
print("[cli]")
|
||||
out = tmp / "cli_index.json"
|
||||
r = subprocess.run([sys.executable, str(HERE / "raglite.py"),
|
||||
"index", str(tmp), "--out", str(out)],
|
||||
capture_output=True, text=True)
|
||||
check("cli index exits 0", r.returncode == 0, r.stderr[-400:])
|
||||
check("cli index reports 8 passages", "8 passages" in r.stdout, r.stdout.strip())
|
||||
|
||||
r = subprocess.run([sys.executable, str(HERE / "raglite.py"),
|
||||
"query", str(out), "elasticsearch logs", "-k", "2", "--json"],
|
||||
capture_output=True, text=True)
|
||||
check("cli query exits 0", r.returncode == 0, r.stderr[-400:])
|
||||
try:
|
||||
hits = json.loads(r.stdout)
|
||||
except json.JSONDecodeError as e:
|
||||
check("cli query emits valid JSON", False, f"{e}: {r.stdout[:200]}")
|
||||
return
|
||||
check("cli query emits valid JSON", True)
|
||||
check("cli top hit is kibana",
|
||||
bool(hits) and doc_of(hits[0]["id"]) == "kibana.md", repr(hits))
|
||||
|
||||
|
||||
def main():
|
||||
print("raglite test suite")
|
||||
test_tokenize()
|
||||
test_split_passages()
|
||||
with tempfile.TemporaryDirectory(prefix="raglite_test_") as d:
|
||||
tmp = Path(d)
|
||||
write_corpus(tmp)
|
||||
test_index_and_ranking(tmp)
|
||||
test_roundtrip(tmp)
|
||||
test_cli(tmp)
|
||||
test_exact_bm25()
|
||||
print()
|
||||
if FAILURES:
|
||||
print(f"FAILED ({len(FAILURES)}): " + ", ".join(FAILURES))
|
||||
return 1
|
||||
print("ALL CHECKS PASSED")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user