diff --git a/wizard-work/null/minisql/_probe_join_null_on.py b/wizard-work/null/minisql/_probe_join_null_on.py new file mode 100644 index 0000000..57e4a57 --- /dev/null +++ b/wizard-work/null/minisql/_probe_join_null_on.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Probe: does `LEFT JOIN ` produce a NULL-extended row in +sqlite3, and does the `and-null-absorbs-false` mutant change any query's answer? + +Reads like a spell-check of a claim. Every number printed is measured here. +""" +import os +import sqlite3 +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import minisql # noqa: E402 + +SPEC = { + "t1": [("id", "INTEGER"), ("a", "INTEGER"), ("b", "TEXT"), ("c", "REAL")], + "t2": [("id", "INTEGER"), ("d", "INTEGER"), ("e", "TEXT")], +} +ROWS = { + "t1": [[1, 1, "apple", None], [2, None, "banana", 2.5], [3, 3, None, 4.0]], + "t2": [[1, 7, "apple"], [2, None, "zzz"], [3, 9, "banana"]], +} +TABLES = {n: minisql.table_from(c, ROWS[n]) for n, c in SPEC.items()} + +QUERIES = [ + ("plain scan", "SELECT id, a FROM t1"), + ("left join plain", "SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id"), + ("left join null-on", + "SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND (t1.a = 100)"), + ("left join null-on2", + "SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND (t2.d = 100)"), + ("left join null-on3", + "SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND (t2.d = 100 AND t2.id > 0)"), + ("left join where-null", + "SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id WHERE t2.d = 100"), + ("left join null-on + where", + "SELECT t1.id, t2.id FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND (t2.d = 100) " + "WHERE t1.a = 1"), +] + + +def run(sql): + con = sqlite3.connect(":memory:") + for name, cols in SPEC.items(): + con.execute("CREATE TABLE %s (%s)" % ( + name, ", ".join("%s %s" % (c, t) for c, t in cols))) + con.executemany("INSERT INTO %s VALUES (%s)" % ( + name, ", ".join("?" * len(cols))), ROWS[name]) + oracle = [list(r) for r in con.execute(sql).fetchall()] + con.close() + engine = minisql.execute(sql, TABLES)[1] + return oracle, engine + + +def main(): + print("A. does sqlite3 produce NULL-extended rows for a NULL-y ON?") + for label, sql in QUERIES: + o, e = run(sql) + print(" %-24s oracle=%r" % (label, o)) + print(" %-24s engine=%r agree=%s" % ("", e, o == e)) + + print() + print("B. and-null-absorbs-false mutant, same query set") + orig = minisql._tri_and + + def m_and_absorb(a, b): + ta, tb = minisql.tri(a), minisql.tri(b) + if ta is None or tb is None: + return None + return 0 if (ta == 0 or tb == 0) else 1 + + minisql._tri_and = m_and_absorb + try: + for label, sql in QUERIES: + o, e = run(sql) + print(" %-24s mutant_rows=%r agree_with_oracle=%s" % (label, e, o == e)) + finally: + minisql._tri_and = orig + print() + print("C. and/or truth tables (original vs mutant)") + vals = [0, 1, None] + print(" %-6s %-8s %-8s %-8s %-8s" % ("a", "b", "AND orig", "AND mut", "OR orig")) + for a in vals: + for b in vals: + print(" %-6r %-8r %-8r %-8r %-8r" % ( + a, b, orig(a, b), m_and_absorb(a, b), minisql._tri_or(a, b))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/null/minisql/diff_sqlite.py b/wizard-work/null/minisql/diff_sqlite.py new file mode 100644 index 0000000..c0edab5 --- /dev/null +++ b/wizard-work/null/minisql/diff_sqlite.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +""" +diff_sqlite.py -- randomized differential battery: minisql.py vs sqlite3 (the oracle). + +Wizard: null (lane: Codeforge -- hard coding and agentic reasoning). + +What it does, per query: + 1. generates a random schema (2-3 tables, INTEGER/TEXT/REAL columns, NULLs, 0-12 rows), + 2. generates one random SELECT in the supported grammar, + 3. creates the SAME schema in an in-memory sqlite3 database and executes the SAME SQL text + in both engines, + 4. compares the result sets -- strictly, by type category (int / real / text / null) and by + value -- as an ordered sequence when the query has ORDER BY, else as a multiset. + +It exits 0 only when ALL of these hold, so the exit code cannot pass vacuously: + * sqlite3 is importable and every generated statement ran on BOTH engines without error, + * the requested number of queries actually executed, + * every query shape was exercised (const / scan / where / join / group / agg / distinct), + * at least a set fraction of queries returned at least one row, and the battery compared at + least a set number of rows (so agreement is not agreement about empty sets), + * zero mismatches, + * with --mutants: every seeded engine defect is CAUGHT (each mutant must produce at least + one mismatch). A mutant that survives means this battery has no teeth for that semantics. + +Generator constraints (each one is there to keep the oracle deterministic, and each is a +documented hole rather than an accident): + * every ORDER BY is made total by appending the `id` columns of the FROM tables, so ties + cannot reorder differently in the two engines; + * DISTINCT queries are never ordered (SQLite requires ORDER BY terms to be in the result + set, and ordered ties would not be comparable) -- they are compared as multisets; + * LIMIT/OFFSET only appear with an ORDER BY; + * with GROUP BY, the select list is group keys plus aggregates only -- never a bare column + of a multi-row group, whose value SQLite leaves unspecified. + +Usage: + python3 diff_sqlite.py [--queries N] [--seed S] [--mutants] [--max-print K] [--quiet] +""" + +from __future__ import annotations + +import argparse +import os +import random +import sqlite3 +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import minisql # noqa: E402 + +SHAPES = ("const", "scan", "where", "join", "group", "agg", "distinct") + +TEXT_POOL = ["apple", "banana", "cherry", "a", "ab", "", "A", "10", "20"] +LIKE_POOL = ["%a%", "a%", "%b", "_", "ab", "A%", "%", "", "c_erry", "ap_le", "%rry"] +NUM_LITERALS = ["0", "1", "-1", "2", "5", "10", "20", "3.5", "5.5", "0.0"] +STR_NUM_LITERALS = ["0", "5", "10", "20", "5.5", "abc", ""] + + +# -------------------------------------------------------------------------- +# schema +# -------------------------------------------------------------------------- + +def make_schema(rng): + """Return (spec, rows): spec is {table: [(col, declared_type)]}, rows is {table: [[v...]]}. + + `id` is a unique non-NULL INTEGER in every table; it is the tiebreaker the generator + appends to every ORDER BY, so ordered results are deterministic in both engines. + """ + spec = { + "t1": [("id", "INTEGER"), ("a", "INTEGER"), ("b", "TEXT"), ("c", "REAL")], + "t2": [("id", "INTEGER"), ("d", "INTEGER"), ("e", "TEXT")], + } + if rng.random() < 0.5: + spec["t3"] = [("id", "INTEGER"), ("f", "REAL"), ("g", "TEXT")] + + rows = {} + for name, cols in spec.items(): + n = rng.randint(0, 12) + table_rows = [] + for i in range(1, n + 1): + row = [] + for cname, ctype in cols: + if cname == "id": + row.append(i) + elif rng.random() < 0.2: + row.append(None) + elif ctype == "INTEGER": + row.append(rng.randint(0, 20)) + elif ctype == "REAL": + row.append(round(rng.uniform(0, 10), 1)) + else: + row.append(rng.choice(TEXT_POOL)) + table_rows.append(row) + rows[name] = table_rows + return spec, rows + + +def load_sqlite(con, spec, rows): + for name, cols in spec.items(): + con.execute("CREATE TABLE %s (%s)" % ( + name, ", ".join("%s %s" % (c, t) for c, t in cols))) + if rows[name]: + con.executemany("INSERT INTO %s VALUES (%s)" % ( + name, ", ".join("?" * len(cols))), rows[name]) + + +# -------------------------------------------------------------------------- +# query generator +# -------------------------------------------------------------------------- + +class Gen: + """Emits one random SELECT. All column references are drawn from the tables named in + `scope`, which the query method sets to exactly the FROM tables before generating.""" + + def __init__(self, rng, spec, rows): + self.rng = rng + self.spec = spec + self.rows = rows + self.tables = list(spec) + self.scope = [] + self.pool_num = [] # [(table, col)] with INTEGER/REAL type, inside scope + self.pool_text = [] # [(table, col)] with TEXT type, inside scope + self.pool_all = [] # [(table, col)] any type except id + self.pool_id = [] # [(table, "id")] + + # -- scope ------------------------------------------------------------ + def set_scope(self, tables): + self.scope = list(tables) + self.pool_num, self.pool_text, self.pool_all, self.pool_id = [], [], [], [] + for t in self.scope: + for c, ty in self.spec[t]: + if c == "id": + self.pool_id.append((t, c)) + else: + self.pool_all.append((t, c)) + (self.pool_num if ty in ("INTEGER", "REAL") else self.pool_text).append((t, c)) + + def ref(self, tc, qualify): + t, c = tc + return "%s.%s" % (t, c) if qualify else c + + def pick(self, pool): + return self.rng.choice(pool) if pool else None + + # -- literals --------------------------------------------------------- + def num_literal(self): + return self.rng.choice(NUM_LITERALS) + + def text_literal(self): + return "'%s'" % self.rng.choice(TEXT_POOL) + + # -- expressions ------------------------------------------------------ + def numeric_expr(self, qualify, depth=0): + pool = self.pool_num + r = self.rng.random() + if not pool or r < 0.4 or depth >= 2: + return self.ref(self.pick(pool), qualify) if pool else self.num_literal() + if r < 0.55: + return self.num_literal() + if r < 0.68: + return "(%s + %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal()) + if r < 0.8: + return "(%s * %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal()) + if r < 0.9: + return "(%s - %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal()) + # division is here on purpose: SQLite's `/` stays INTEGER for two integers, and the + # `int-div-becomes-real` mutant exists to prove this battery notices when it does not. + return "(%s / %s)" % (self.numeric_expr(qualify, depth + 1), + self.rng.choice(["2", "3", "1", "4"])) + + def scalar_expr(self, qualify, depth=0): + r = self.rng.random() + if r < 0.45: + return self.numeric_expr(qualify, depth) + if r < 0.6 and self.pool_text: + return self.ref(self.pick(self.pool_text), qualify) + if r < 0.7: + return self.text_literal() + if r < 0.8 and depth < 2: + return "CASE WHEN %s THEN %s ELSE %s END" % ( + self.predicate(qualify, depth + 1), self.scalar_expr(qualify, depth + 1), + self.scalar_expr(qualify, depth + 1)) + return self.predicate(qualify, depth + 1) + + def predicate(self, qualify, depth=0): + r = self.rng.random() + if depth >= 2: + r = min(r, 0.59) + if r < 0.35: + return self.comparison(qualify) + if r < 0.45: + return "(%s AND %s)" % (self.predicate(qualify, depth + 1), + self.predicate(qualify, depth + 1)) + if r < 0.55: + return "(%s OR %s)" % (self.predicate(qualify, depth + 1), + self.predicate(qualify, depth + 1)) + if r < 0.6: + return "NOT %s" % self.predicate(qualify, depth + 1) + if r < 0.68 and self.pool_all: + tc = self.pick(self.pool_all) + return "%s IS%s NULL" % (self.ref(tc, qualify), + "" if self.rng.random() < 0.5 else " NOT") + if r < 0.82 and self.pool_num: + items = ", ".join(self.num_literal() for _ in range(self.rng.randint(1, 3))) + return "%s%s IN (%s)" % (self.ref(self.pick(self.pool_num), qualify), + " NOT" if self.rng.random() < 0.3 else "", items) + if r < 0.88 and self.pool_text: + items = ", ".join(self.text_literal() for _ in range(self.rng.randint(1, 3))) + return "%s%s IN (%s)" % (self.ref(self.pick(self.pool_text), qualify), + " NOT" if self.rng.random() < 0.3 else "", items) + if r < 0.93 and self.pool_num: + return "%s%s BETWEEN %s AND %s" % ( + self.ref(self.pick(self.pool_num), qualify), + " NOT" if self.rng.random() < 0.3 else "", + self.num_literal(), self.num_literal()) + if self.pool_text: + return "%s%s LIKE '%s'" % (self.ref(self.pick(self.pool_text), qualify), + " NOT" if self.rng.random() < 0.3 else "", + self.rng.choice(LIKE_POOL)) + return self.comparison(qualify) + + def comparison(self, qualify): + """The mixed-affinity forms are deliberate: they are where an engine that ignores + SQLite's two affinity rules diverges from it.""" + r = self.rng.random() + op = self.rng.choice(["=", "<>", "<", "<=", ">", ">="]) + if r < 0.35 and self.pool_num: + return "%s %s %s" % (self.ref(self.pick(self.pool_num), qualify), op, + self.num_literal()) + if r < 0.55 and self.pool_text: + return "%s %s %s" % (self.ref(self.pick(self.pool_text), qualify), op, + self.text_literal()) + if r < 0.7 and self.pool_num: + # numeric affinity is applied to the text literal + return "%s %s '%s'" % (self.ref(self.pick(self.pool_num), qualify), op, + self.rng.choice(STR_NUM_LITERALS)) + if r < 0.85 and self.pool_text: + # TEXT affinity is applied to the numeric literal + return "%s %s %s" % (self.ref(self.pick(self.pool_text), qualify), op, + self.rng.choice(["0", "5", "10", "20", "-1"])) + if self.pool_num: + return "%s %s %s" % (self.numeric_expr(qualify, 1), op, self.num_literal()) + return self.comparison(qualify) if (self.pool_text or self.pool_num) else "1 = 1" + + # -- select items ----------------------------------------------------- + def agg_item(self, qualify, i): + fn = self.rng.choice(["COUNT", "SUM", "AVG", "MIN", "MAX"]) + if fn == "COUNT" and self.rng.random() < 0.4: + return "COUNT(*) AS agg%d" % i + pool = self.pool_num if (fn in ("SUM", "AVG") and self.pool_num) else self.pool_all + if not pool: + return "COUNT(*) AS agg%d" % i + return "%s(%s) AS agg%d" % (fn, self.ref(self.pick(pool), qualify), i) + + def items(self, qualify, agg=False, group_keys=None): + out = [] + if not agg and self.rng.random() < 0.25: + out.append("*") + for i in range(self.rng.randint(1, 3)): + if agg: + if group_keys and self.rng.random() < 0.35: + out.append("%s AS gk%d" % (self.rng.choice(group_keys), i)) + else: + out.append(self.agg_item(qualify, i)) + else: + e = self.scalar_expr(qualify) + out.append("%s AS x%d" % (e, i) if self.rng.random() < 0.6 else e) + if not out: + out.append("*") + return out + + def having(self, gkeys): + r = self.rng.random() + if r < 0.45: + return "COUNT(*) > %d" % self.rng.randint(0, 3) + if r < 0.6 and self.pool_num: + return "SUM(%s) > %s" % (self.ref(self.pick(self.pool_num), False), + self.rng.choice(["5", "10", "20", "0"])) + if r < 0.7 and self.pool_num: + return "AVG(%s) > %s" % (self.ref(self.pick(self.pool_num), False), + self.rng.choice(["1", "5", "10"])) + if r < 0.8 and self.pool_all: + return "MAX(%s) > %s" % (self.ref(self.pick(self.pool_all), False), + self.rng.choice(["5", "10", "1.5"])) + if r < 0.9 and gkeys: + return "%s IS NOT NULL" % self.rng.choice(gkeys) + return "COUNT(*) > 1" + + def order_key(self, qualify): + """ORDER BY terms are always column references: a bare integer literal in ORDER BY is + an ORDINAL in SQLite, not a constant, and the oracle rejects it out of range.""" + pool = list(self.pool_all) + list(self.pool_id) + if not pool: + return "1 = 1" + return self.ref(self.rng.choice(pool), qualify) + + def order_tail(self, keys, tiebreak): + """keys: ordering exprs; tiebreak: exprs that make the order total.""" + terms = ["%s%s" % (k, " DESC" if self.rng.random() < 0.4 else "") for k in keys] + terms.extend(tiebreak) + tail = " ORDER BY " + ", ".join(terms) + if self.rng.random() < 0.35: + tail += " LIMIT %d" % self.rng.randint(0, 6) + if self.rng.random() < 0.4: + tail += " OFFSET %d" % self.rng.randint(0, 4) + return tail + + # -- whole queries ---------------------------------------------------- + def query(self, shape): + rng = self.rng + t = rng.choice(self.tables) + + if shape == "const": + self.set_scope([]) + return "SELECT " + ", ".join(self.scalar_expr(False) for _ in range(rng.randint(1, 3))) + + if shape in ("scan", "where", "distinct"): + self.set_scope([t]) + sql = "SELECT " + ("DISTINCT " if shape == "distinct" else "") \ + + ", ".join(self.items(False)) + sql += " FROM " + t + if shape in ("where", "distinct") or rng.random() < 0.4: + sql += " WHERE " + self.predicate(False) + if shape != "distinct": + keys = [self.order_key(False) for _ in range(rng.randint(1, 2))] + sql += self.order_tail(keys, ["%s.id" % t]) + return sql + + if shape == "join": + others = [x for x in self.tables if x != t] + if not others: + return self.query("scan") + u = rng.choice(others) + self.set_scope([t, u]) + kind = "LEFT JOIN" if rng.random() < 0.45 else "JOIN" + cands = [c for (tt, c) in self.pool_num if tt == t] or ["id"] + on = rng.choice([ + "%s.id = %s.id" % (t, u), + "%s.id = %s.id AND %s" % (t, u, self.predicate(True)), + "%s.%s = %s.id" % (t, rng.choice(cands), u), + ]) + grouped = rng.random() < 0.5 + if grouped: + gkeys = ["%s.id" % t, "%s.id" % u] + sql = "SELECT " + ", ".join(self.items(True, agg=True, group_keys=gkeys)) + sql += " FROM %s %s %s ON %s" % (t, kind, u, on) + if rng.random() < 0.5: + sql += " WHERE " + self.predicate(True) + sql += " GROUP BY " + ", ".join(gkeys) + sql += self.order_tail(["COUNT(*)"], gkeys) + else: + sql = "SELECT " + ", ".join(self.items(True, agg=False)) + sql += " FROM %s %s %s ON %s" % (t, kind, u, on) + if rng.random() < 0.5: + sql += " WHERE " + self.predicate(True) + sql += self.order_tail(["%s.id" % t], ["%s.id" % t, "%s.id" % u]) + return sql + + if shape == "agg": + self.set_scope([t]) + sql = "SELECT " + ", ".join(self.items(False, agg=True)) + sql += " FROM " + t + if rng.random() < 0.6: + sql += " WHERE " + self.predicate(False) + if rng.random() < 0.4: + sql += " HAVING " + self.having(None) + return sql + + # group + self.set_scope([t]) + gkeys = [] + for _ in range(rng.randint(1, 2)): + if self.pool_all: + gkeys.append(self.ref(self.pick(self.pool_all), False)) + if not gkeys: + gkeys = ["%s.id" % t] + sql = "SELECT " + ", ".join(self.items(False, agg=True, group_keys=gkeys)) + sql += " FROM " + t + if rng.random() < 0.6: + sql += " WHERE " + self.predicate(False) + sql += " GROUP BY " + ", ".join(gkeys) + if rng.random() < 0.5: + sql += " HAVING " + self.having(gkeys) + sql += self.order_tail(["COUNT(*)"], gkeys) + return sql + + +# -------------------------------------------------------------------------- +# comparison +# -------------------------------------------------------------------------- + +def cat(v): + if v is None: + return "null" + if isinstance(v, bool): + return "int" + if isinstance(v, int): + return "int" + if isinstance(v, float): + return "real" + if isinstance(v, str): + return "text" + return "other:" + type(v).__name__ + + +def same_value(a, b): + return cat(a) == cat(b) and a == b + + +def canon_row(row): + out = [] + for v in row: + c = cat(v) + out.append(("num", float(v)) if c in ("int", "real") else (c, str(v))) + return tuple(out) + + +def same_row(ra, rb): + return len(ra) == len(rb) and all(same_value(a, b) for a, b in zip(ra, rb)) + + +def compare_rows(got, want, ordered): + """None when the two result sets agree, else a short description of the first difference.""" + if len(got) != len(want): + return "row count %d != %d" % (len(got), len(want)) + if ordered: + for i, (g, w) in enumerate(zip(got, want)): + if not same_row(g, w): + return "row %d: %r != %r" % (i, g, w) + return None + for g, w in zip(sorted(got, key=canon_row), sorted(want, key=canon_row)): + if not same_row(g, w): + return "multiset row %r != %r" % (g, w) + return None + + +# -------------------------------------------------------------------------- +# runner +# -------------------------------------------------------------------------- + +def run_battery(queries, seed, max_print=5, quiet=False): + rng = random.Random(seed) + con = sqlite3.connect(":memory:") + shape_counts = dict.fromkeys(SHAPES, 0) + mismatches, errors = [], [] + rows_compared = nonempty = ordered_count = 0 + + for i in range(queries): + shape = SHAPES[i % len(SHAPES)] + spec, rows = make_schema(rng) + con.close() + con = sqlite3.connect(":memory:") + load_sqlite(con, spec, rows) + mtables = {name: minisql.table_from(cols, rows[name]) for name, cols in spec.items()} + sql = Gen(rng, spec, rows).query(shape) + shape_counts[shape] += 1 + ordered = " ORDER BY " in sql + ordered_count += 1 if ordered else 0 + try: + want = [list(r) for r in con.execute(sql).fetchall()] + except sqlite3.Error as exc: + errors.append((sql, str(exc))) + continue + try: + _names, got = minisql.execute(sql, mtables) + except Exception as exc: # noqa: BLE001 - any engine failure is red + mismatches.append((shape, sql, "minisql raised %s: %s" % (type(exc).__name__, exc), + None, want)) + continue + rows_compared += len(want) + nonempty += 1 if want else 0 + diff = compare_rows(got, want, ordered) + if diff is not None: + mismatches.append((shape, sql, diff, got, want)) + con.close() + + report = { + "queries": queries, "seed": seed, "shape_counts": shape_counts, + "rows_compared": rows_compared, "nonempty": nonempty, "ordered": ordered_count, + "mismatches": mismatches, "errors": errors, + } + if not quiet: + print("minisql differential battery vs sqlite3 %s" % sqlite3.sqlite_version) + print("seed=%d queries=%d" % (seed, queries)) + print("shape counts: " + " ".join("%s=%d" % (k, shape_counts[k]) for k in SHAPES)) + print("ordered queries: %d rows compared: %d queries returning >=1 row: %d" + % (ordered_count, rows_compared, nonempty)) + for shape, sql, diff, got, want in mismatches[:max_print]: + print("\nMISMATCH [%s] %s\n %s" % (shape, diff, sql)) + print(" oracle: %r" % (want[:6],)) + print(" engine: %r" % (got[:6] if got is not None else None,)) + if len(mismatches) > max_print: + print("... %d more mismatches" % (len(mismatches) - max_print)) + for sql, exc in errors[:max_print]: + print("\nHARNESS ERROR (oracle refused generated SQL): %s\n %s" % (exc, sql)) + if len(errors) > max_print: + print("... %d more harness errors" % (len(errors) - max_print)) + return report + + +def check_report(report): + """The non-vacuity guards. Returns a list of failure strings.""" + bad = [] + if report["errors"]: + bad.append("%d generated statements were refused by the oracle (harness defect)" + % len(report["errors"])) + if report["mismatches"]: + bad.append("%d mismatches" % len(report["mismatches"])) + if report["queries"] < len(SHAPES): + bad.append("too few queries to exercise every shape") + for shape in SHAPES: + if report["shape_counts"][shape] == 0: + bad.append("shape %s was never exercised" % shape) + if report["nonempty"] < 0.5 * report["queries"]: + bad.append("only %d/%d queries returned rows -- agreement would be too close to vacuous" + % (report["nonempty"], report["queries"])) + if report["rows_compared"] < report["queries"]: + bad.append("only %d rows compared for %d queries" + % (report["rows_compared"], report["queries"])) + if report["ordered"] < 0.2 * report["queries"]: + bad.append("only %d ordered queries -- ORDER BY is barely covered" % report["ordered"]) + return bad + + +# -------------------------------------------------------------------------- +# mutation control: prove the battery can go RED +# -------------------------------------------------------------------------- + +def _mutants(): + orig = { + "compare_values": minisql.compare_values, + "join_relation": minisql.join_relation, + "sort_key": minisql.sort_key, + "agg_sum": minisql.agg_sum, + "agg_count": minisql.agg_count, + "arith_div": minisql.arith_div, + "_tri_and": minisql._tri_and, + "like_match": minisql.like_match, + } + + def m_null_eq(a, b, aff_a=None, aff_b=None): + if a is None or b is None: + return 0 # NULL = NULL is TRUE, and NULL = 1 is TRUE + return orig["compare_values"](a, b, aff_a, aff_b) + + def m_left_inner(left, right, kind, on_eval): + return orig["join_relation"](left, right, "inner", on_eval) + + def m_nulls_last(v): + if v is None: + return (3, 0) # NULLs sort last on ASC + return (min(minisql.class_rank(v), 2), v) + + def m_sum_empty(values): + r = orig["agg_sum"](values) + return 0 if r is None else r + + def m_count_nulls(values, n_rows): + return n_rows # COUNT(x) counts NULLs too + + def m_div_float(a, b): + r = orig["arith_div"](a, b) + return None if r is None else float(r) + + def m_and_absorb(a, b): + ta, tb = minisql.tri(a), minisql.tri(b) + if ta is None or tb is None: + return None # NULL absorbs FALSE + return 0 if (ta == 0 or tb == 0) else 1 + + def m_like_case_sensitive(value, pattern): + if value is None or pattern is None: + return None + import re as _re + rx = _re.compile("(?s)\\A" + "".join( + ".*" if ch == "%" else ("." if ch == "_" else _re.escape(ch)) + for ch in pattern) + "\\Z") + return 1 if rx.match(str(value)) else 0 + + return [ + ("null-eq", "compare_values", m_null_eq), + ("left-join-as-inner", "join_relation", m_left_inner), + ("nulls-last-asc", "sort_key", m_nulls_last), + ("sum-empty-zero", "agg_sum", m_sum_empty), + ("count-counts-nulls", "agg_count", m_count_nulls), + ("int-div-becomes-real", "arith_div", m_div_float), + ("and-null-absorbs-false", "_tri_and", m_and_absorb), + ("like-case-sensitive", "like_match", m_like_case_sensitive), + ] + + +def run_mutants(queries, seed, quiet=False): + results = [] + for name, attr, fn in _mutants(): + original = getattr(minisql, attr) + setattr(minisql, attr, fn) + try: + rep = run_battery(queries, seed, max_print=0, quiet=True) + finally: + setattr(minisql, attr, original) + caught = len(rep["mismatches"]) + results.append((name, attr, caught)) + if not quiet: + print(" mutant %-24s patches %-15s -> %s" + % (name, attr, "CAUGHT (%d mismatches)" % caught if caught + else "SURVIVED -- battery has no teeth here")) + return results + + +# -------------------------------------------------------------------------- + +def main(argv=None): + ap = argparse.ArgumentParser(description="minisql vs sqlite3 differential battery") + ap.add_argument("--queries", type=int, default=1400) + ap.add_argument("--seed", type=int, default=20261022) + ap.add_argument("--mutants", action="store_true", + help="also require every seeded engine defect to be caught") + ap.add_argument("--mutant-queries", type=int, default=140) + ap.add_argument("--max-print", type=int, default=5) + ap.add_argument("--quiet", action="store_true") + args = ap.parse_args(argv) + + rep = run_battery(args.queries, args.seed, max_print=args.max_print, quiet=args.quiet) + bad = check_report(rep) + + mutants_caught = 0 + if args.mutants: + if not args.quiet: + print("\nmutation control (every seeded defect must be caught):") + results = run_mutants(args.mutant_queries, args.seed, quiet=args.quiet) + survived = [n for n, _a, c in results if not c] + mutants_caught = len(results) - len(survived) + if not args.quiet: + print("mutants caught: %d/%d" % (mutants_caught, len(results))) + if survived: + bad.append("mutants survived (battery has no teeth): " + ", ".join(survived)) + + if not args.quiet: + print() + for b in bad: + print("FAIL: " + b) + if not bad: + print("all %d queries agreed with sqlite3; %d rows compared; %d mutants caught" + % (rep["queries"], rep["rows_compared"], mutants_caught)) + print("RESULT: " + ("FAIL" if bad else "PASS")) + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/null/minisql/minisql.py b/wizard-work/null/minisql/minisql.py new file mode 100644 index 0000000..9b45678 --- /dev/null +++ b/wizard-work/null/minisql/minisql.py @@ -0,0 +1,1234 @@ +#!/usr/bin/env python3 +""" +minisql.py -- a from-scratch SQL SELECT engine. Stdlib only, no third-party imports. + +Wizard: null (lane: Codeforge -- hard coding and agentic reasoning). + +Scope: exactly the grammar the differential harness `diff_sqlite.py` generates, and +nothing else. Everything outside it raises SqlError. + + SELECT [DISTINCT] FROM [[AS] alias] + [[INNER|LEFT [OUTER]] JOIN
[[AS] alias] ON ]* + [WHERE ] + [GROUP BY , ...] + [HAVING ] + [ORDER BY [ASC|DESC], ...] + [LIMIT [OFFSET ]] + + items : * | alias.* | [[AS] alias] + expr : literals (int, real, 'text', NULL, TRUE, FALSE) + | column refs (name | alias.name) + | unary - + NOT ; binary + - * / + | = <> != < <= > >= (three-valued) + | IS [NOT] NULL | [NOT] IN (list) | [NOT] BETWEEN a AND b | [NOT] LIKE 'pat' + | AND OR | ( expr ) + | CASE WHEN c THEN e [...] [ELSE e] END (searched CASE only) + | COUNT(*) COUNT(x) SUM(x) AVG(x) MIN(x) MAX(x) + +Semantics implemented deliberately to match SQLite 3.x: + * NULL propagation, three-valued AND/OR/NOT; WHERE keeps rows whose value is true. + * storage-class ordering NULL < INTEGER/REAL < TEXT, and the affinity rule: if one + operand has INTEGER/REAL/NUMERIC affinity and the other has TEXT/BLOB/no affinity, + numeric affinity is applied to the other operand. + * integer division truncates toward zero and stays INTEGER; division by zero -> NULL. + * SUM of no non-NULL values -> NULL (not 0); COUNT(x) ignores NULLs; AVG -> REAL. + * LIKE is case-insensitive for ASCII only, `%` = any run, `_` = one char. + * ASC sorts NULLs first, DESC last; comparison is by storage class then value. + +The module is written so each semantic decision above lives in its own named function +(compare_values, sort_key, arith_div, agg_sum, agg_count, distinct_key, join_relation, +tri/truthy), because diff_sqlite.py mutates them one at a time to prove its own teeth. +""" + +from __future__ import annotations + +import re +import sys +from dataclasses import dataclass, replace +from typing import Optional + +__all__ = [ + "SqlError", "parse_query", "execute", "run", "Table", + "compare_values", "sort_key", "arith_div", "agg_sum", "agg_count", + "distinct_key", "tri", "truthy", "like_match", +] + + +class SqlError(Exception): + """Raised for anything outside the supported subset.""" + + +# -------------------------------------------------------------------------- +# tokenizer +# -------------------------------------------------------------------------- + +_KEYWORDS = { + "SELECT", "DISTINCT", "FROM", "JOIN", "INNER", "LEFT", "OUTER", "ON", "WHERE", + "GROUP", "BY", "HAVING", "ORDER", "ASC", "DESC", "LIMIT", "OFFSET", "AS", + "AND", "OR", "NOT", "IS", "NULL", "IN", "BETWEEN", "LIKE", "CASE", "WHEN", + "THEN", "ELSE", "END", "TRUE", "FALSE", +} +_AGGS = {"COUNT", "SUM", "AVG", "MIN", "MAX"} + +_TOKEN_RE = re.compile( + r""" + (?P\s+) + | (?P--[^\n]*) + | (?P'(?:[^']|'')*') + | (?P"(?:[^"]|"")*") + | (?P\d+\.\d*(?:[eE][-+]?\d+)?|\.\d+(?:[eE][-+]?\d+)?|\d+(?:[eE][-+]?\d+)?) + | (?P[A-Za-z_][A-Za-z_0-9]*) + | (?P<=|>=|<>|!=|\|\||[-+*/%(),.=<>;]) + """, + re.VERBOSE, +) + + +@dataclass +class _Tok: + kind: str # 'kw' | 'ident' | 'num' | 'str' | 'op' | 'end' + text: str # canonical text (keywords uppercased, idents lowercased) + value: object = None + + +def tokenize(sql: str) -> list: + toks, i, n = [], 0, len(sql) + while i < n: + m = _TOKEN_RE.match(sql, i) + if not m: + raise SqlError("cannot tokenize at offset %d: %r" % (i, sql[i:i + 12])) + i = m.end() + kind = m.lastgroup + text = m.group() + if kind in ("ws", "comment"): + continue + if kind == "str": + toks.append(_Tok("str", text, text[1:-1].replace("''", "'"))) + elif kind == "qident": + toks.append(_Tok("ident", text[1:-1].replace('""', '"').lower())) + elif kind == "num": + if "." in text or "e" in text or "E" in text: + toks.append(_Tok("num", text, float(text))) + else: + toks.append(_Tok("num", text, int(text))) + elif kind == "ident": + up = text.upper() + if up in _KEYWORDS: + toks.append(_Tok("kw", up)) + else: + toks.append(_Tok("ident", text.lower())) + else: + toks.append(_Tok("op", text)) + toks.append(_Tok("end", "")) + return toks + + +# -------------------------------------------------------------------------- +# parser +# -------------------------------------------------------------------------- +# AST nodes are plain tuples; the first element names the node. + +class _Parser: + def __init__(self, toks): + self.t = toks + self.i = 0 + + # -- helpers ---------------------------------------------------------- + def peek(self, k=0): + return self.t[min(self.i + k, len(self.t) - 1)] + + def next(self): + tok = self.t[self.i] + self.i += 1 + return tok + + def at_kw(self, *names): + tok = self.peek() + return tok.kind == "kw" and tok.text in names + + def at_op(self, *ops): + tok = self.peek() + return tok.kind == "op" and tok.text in ops + + def eat_kw(self, name): + if not self.at_kw(name): + raise SqlError("expected %s, got %r" % (name, self.peek().text)) + return self.next() + + def eat_op(self, op): + if not self.at_op(op): + raise SqlError("expected %r, got %r" % (op, self.peek().text)) + return self.next() + + def eat_ident(self): + tok = self.peek() + if tok.kind != "ident": + raise SqlError("expected identifier, got %r" % tok.text) + return self.next().text + + # -- statements ------------------------------------------------------- + def parse_query(self): + self.eat_kw("SELECT") + distinct = False + if self.at_kw("DISTINCT"): + self.next() + distinct = True + items = [self.parse_item()] + while self.at_op(","): + self.next() + items.append(self.parse_item()) + if not self.at_kw("FROM"): + # SELECT without FROM: one row, no columns. Constant-expression queries only. + if self.peek().kind != "end": + raise SqlError("a SELECT without FROM takes no clauses") + return { + "distinct": distinct, "items": items, + "from": (None, None), "joins": [], "where": None, "group_by": [], + "having": None, "order_by": [], "limit": None, "offset": None, + } + self.next() + from_tbl, from_alias = self.parse_table_ref() + joins = [] + while True: + if self.at_kw("JOIN"): + self.next() + kind = "inner" + elif self.at_kw("INNER"): + self.next() + self.eat_kw("JOIN") + kind = "inner" + elif self.at_kw("LEFT"): + self.next() + if self.at_kw("OUTER"): + self.next() + self.eat_kw("JOIN") + kind = "left" + else: + break + tbl, alias = self.parse_table_ref() + self.eat_kw("ON") + on = self.parse_expr() + joins.append((kind, tbl, alias, on)) + where = None + if self.at_kw("WHERE"): + self.next() + where = self.parse_expr() + group_by = [] + if self.at_kw("GROUP"): + self.next() + self.eat_kw("BY") + group_by = [self.parse_expr()] + while self.at_op(","): + self.next() + group_by.append(self.parse_expr()) + having = None + if self.at_kw("HAVING"): + self.next() + having = self.parse_expr() + order_by = [] + if self.at_kw("ORDER"): + self.next() + self.eat_kw("BY") + while True: + e = self.parse_expr() + desc = False + if self.at_kw("ASC"): + self.next() + elif self.at_kw("DESC"): + self.next() + desc = True + order_by.append((e, desc)) + if self.at_op(","): + self.next() + continue + break + limit = offset = None + if self.at_kw("LIMIT"): + self.next() + limit = self.parse_signed_int() + if self.at_kw("OFFSET"): + self.next() + offset = self.parse_signed_int() + if self.peek().kind != "end": + raise SqlError("trailing input: %r" % self.peek().text) + return { + "distinct": distinct, "items": items, + "from": (from_tbl, from_alias), "joins": joins, + "where": where, "group_by": group_by, "having": having, + "order_by": order_by, "limit": limit, "offset": offset, + } + + def parse_signed_int(self): + neg = False + if self.at_op("-"): + self.next() + neg = True + tok = self.next() + if tok.kind != "num" or not isinstance(tok.value, int): + raise SqlError("expected integer, got %r" % tok.text) + return -tok.value if neg else tok.value + + def parse_table_ref(self): + name = self.eat_ident() + alias = name + if self.at_kw("AS"): + self.next() + alias = self.eat_ident() + elif self.peek().kind == "ident": + alias = self.eat_ident() + return name, alias + + def parse_item(self): + if self.at_op("*"): + self.next() + return ("star", None, None) + if self.peek().kind == "ident" and self.peek(1).kind == "op" and self.peek(1).text == "." \ + and self.peek(2).kind == "op" and self.peek(2).text == "*": + tbl = self.next().text + self.next() + self.next() + return ("star", tbl, None) + e = self.parse_expr() + alias = None + if self.at_kw("AS"): + self.next() + alias = self.eat_ident() + elif self.peek().kind == "ident": + alias = self.eat_ident() + return ("item", e, alias) + + # -- expressions ------------------------------------------------------ + def parse_expr(self): + return self.parse_or() + + def parse_or(self): + left = self.parse_and() + while self.at_kw("OR"): + self.next() + left = ("or", left, self.parse_and()) + return left + + def parse_and(self): + left = self.parse_not() + while self.at_kw("AND"): + self.next() + left = ("and", left, self.parse_not()) + return left + + def parse_not(self): + if self.at_kw("NOT"): + self.next() + return ("not", self.parse_not()) + return self.parse_predicate() + + def parse_predicate(self): + left = self.parse_additive() + while True: + if self.at_op("=", "<>", "!=", "<", "<=", ">", ">="): + op = self.next().text + right = self.parse_additive() + left = ("cmp", op, left, right) + elif self.at_kw("IS"): + self.next() + neg = False + if self.at_kw("NOT"): + self.next() + neg = True + self.eat_kw("NULL") + left = ("isnull", left, neg) + elif self.at_kw("IN") or (self.at_kw("NOT") and self.peek(1).kind == "kw" + and self.peek(1).text == "IN"): + neg = False + if self.at_kw("NOT"): + self.next() + neg = True + self.eat_kw("IN") + self.eat_op("(") + items = [] + if not self.at_op(")"): + items.append(self.parse_expr()) + while self.at_op(","): + self.next() + items.append(self.parse_expr()) + self.eat_op(")") + left = ("in", left, items, neg) + elif self.at_kw("BETWEEN") or (self.at_kw("NOT") and self.peek(1).kind == "kw" + and self.peek(1).text == "BETWEEN"): + neg = False + if self.at_kw("NOT"): + self.next() + neg = True + self.eat_kw("BETWEEN") + lo = self.parse_additive() + self.eat_kw("AND") + hi = self.parse_additive() + left = ("between", left, lo, hi, neg) + elif self.at_kw("LIKE") or (self.at_kw("NOT") and self.peek(1).kind == "kw" + and self.peek(1).text == "LIKE"): + neg = False + if self.at_kw("NOT"): + self.next() + neg = True + self.eat_kw("LIKE") + left = ("like", left, self.parse_additive(), neg) + else: + return left + + def parse_additive(self): + left = self.parse_multiplicative() + while self.at_op("+", "-"): + op = self.next().text + left = ("binop", op, left, self.parse_multiplicative()) + return left + + def parse_multiplicative(self): + left = self.parse_unary() + while self.at_op("*", "/", "%"): + op = self.next().text + if op == "%": + raise SqlError("`%` is out of scope") + left = ("binop", op, left, self.parse_unary()) + return left + + def parse_unary(self): + if self.at_op("-"): + self.next() + return ("unary", "-", self.parse_unary()) + if self.at_op("+"): + self.next() + return ("unary", "+", self.parse_unary()) + return self.parse_primary() + + def parse_primary(self): + tok = self.peek() + if tok.kind == "op" and tok.text == "(": + self.next() + e = self.parse_expr() + self.eat_op(")") + return e + if tok.kind == "num": + self.next() + return ("lit", tok.value) + if tok.kind == "str": + self.next() + return ("lit", tok.value) + if tok.kind == "kw" and tok.text == "NULL": + self.next() + return ("lit", None) + if tok.kind == "kw" and tok.text in ("TRUE", "FALSE"): + self.next() + return ("lit", 1 if tok.text == "TRUE" else 0) + if tok.kind == "kw" and tok.text == "CASE": + return self.parse_case() + if tok.kind == "ident": + if tok.text.upper() in _AGGS and self.peek(1).kind == "op" and self.peek(1).text == "(": + return self.parse_aggregate() + name = self.next().text + if self.at_op("."): + self.next() + col = self.eat_ident() + return ("col", name, col) + return ("col", None, name) + raise SqlError("unexpected token %r" % tok.text) + + def parse_aggregate(self): + fname = self.next().text.upper() + self.eat_op("(") + if self.at_op("*"): + self.next() + self.eat_op(")") + if fname != "COUNT": + raise SqlError("%s(*) is not a function" % fname) + return ("agg", "COUNT", None) + arg = self.parse_expr() + self.eat_op(")") + return ("agg", fname, arg) + + def parse_case(self): + self.eat_kw("CASE") + if not self.at_kw("WHEN"): + raise SqlError("only searched CASE is in scope") + whens = [] + while self.at_kw("WHEN"): + self.next() + cond = self.parse_expr() + self.eat_kw("THEN") + val = self.parse_expr() + whens.append((cond, val)) + els = None + if self.at_kw("ELSE"): + self.next() + els = self.parse_expr() + self.eat_kw("END") + return ("case", whens, els) + + +def parse_query(sql: str) -> dict: + return _Parser(tokenize(sql)).parse_query() + + +# -------------------------------------------------------------------------- +# semantics -- each decision below is a separate function on purpose +# -------------------------------------------------------------------------- + +_NUM_AFF = {"INTEGER", "REAL", "NUMERIC"} + + +def class_rank(v): + """SQLite storage-class order: NULL < numbers < text < blob.""" + if v is None: + return 0 + if isinstance(v, (int, float)) and not isinstance(v, bool): + return 1 + if isinstance(v, str): + return 2 + raise SqlError("unsupported value type %r" % (type(v),)) + + +_NUM_PREFIX_RE = re.compile(r"[ \t]*[+-]?(?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?") + + +def numeric_prefix(v): + """SQLite's arithmetic coercion of text: the longest numeric *prefix*, else integer 0. + + Measured against sqlite3 3.53.1: `'5abc' + 0` -> 5 (integer), `'5.5' + 0` -> 5.5 (real), + `'abc' + 0` -> 0 (integer), `'' + 0` -> 0, `' 5 ' + 0` -> 5. Note this is a *prefix* + parse and is NOT the same rule as affinity conversion, which needs the whole string. + """ + if isinstance(v, str): + m = _NUM_PREFIX_RE.match(v) + if not m: + return 0 + text = m.group() + if "." in text or "e" in text or "E" in text: + return float(text) + return int(text) + return v + + +def apply_numeric_affinity(v): + """SQLite numeric affinity: a text value that looks wholly numeric becomes a number.""" + if isinstance(v, str): + s = v.strip() + if not s: + return v + try: + return int(s) + except ValueError: + pass + try: + return float(s) + except ValueError: + return v + return v + + +def apply_text_affinity(v): + """SQLite's TEXT affinity conversion: a number compared against a TEXT-affinity column + becomes its text form. Measured: `s = 10` is TRUE for s='10' while `s = 10.0` is FALSE + (the real converts to '10.0'), so a float must render with SQLite's own shortest form.""" + if v is None or isinstance(v, str): + return v + if isinstance(v, bool): + return str(int(v)) + if isinstance(v, int): + return str(v) + if isinstance(v, float): + if v == int(v) and abs(v) < 1e16: + return "%.1f" % v + return repr(v) + raise SqlError("cannot apply TEXT affinity to %r" % (type(v),)) + + +def compare_values(a, b, aff_a=None, aff_b=None): + """-1/0/1, or None when either side is NULL. Applies SQLite's two affinity rules first: + + * one side INTEGER/REAL/NUMERIC affinity, the other TEXT/BLOB/none -> numeric affinity + is applied to the other side; + * one side TEXT affinity, the other none -> TEXT affinity is applied to the other side. + """ + if a is None or b is None: + return None + if aff_a in _NUM_AFF and aff_b not in _NUM_AFF: + b = apply_numeric_affinity(b) + elif aff_b in _NUM_AFF and aff_a not in _NUM_AFF: + a = apply_numeric_affinity(a) + elif aff_a == "TEXT" and aff_b is None: + b = apply_text_affinity(b) + elif aff_b == "TEXT" and aff_a is None: + a = apply_text_affinity(a) + ra, rb = class_rank(a), class_rank(b) + if ra != rb: + return -1 if ra < rb else 1 + if a == b: + return 0 + return -1 if a < b else 1 + + +def sort_key(v): + """Ordering key for ORDER BY / MIN / MAX: class rank then value (NULLs first ascending).""" + r = class_rank(v) + return (r, 0 if v is None else v) + + +def tri(v): + """Truth value of an expression as 1/0/None (three-valued).""" + if v is None: + return None + if isinstance(v, bool): + return 1 if v else 0 + if isinstance(v, (int, float)): + return 1 if v != 0 else 0 + if isinstance(v, str): + n = apply_numeric_affinity(v) + if isinstance(n, (int, float)) and not isinstance(n, bool): + return 1 if n != 0 else 0 + return 0 + raise SqlError("cannot test truth of %r" % (type(v),)) + + +def truthy(v): + return tri(v) == 1 + + +def arith_div(a, b): + """SQLite `/`: integer when both are integers, truncating toward zero; x/0 -> NULL.""" + if a is None or b is None: + return None + a, b = numeric_prefix(a), numeric_prefix(b) + if b == 0: + return None + if isinstance(a, int) and isinstance(b, int): + q = abs(a) // abs(b) + return -q if (a < 0) != (b < 0) else q + return a / b + + +def arith(op, a, b): + if a is None or b is None: + return None + a, b = numeric_prefix(a), numeric_prefix(b) + if op == "/": + return arith_div(a, b) + if op == "+": + return a + b + if op == "-": + return a - b + if op == "*": + return a * b + raise SqlError("unknown operator %r" % op) + + +def agg_numeric(v): + """Aggregate coercion: numbers pass, text becomes numeric only if the WHOLE string is + numeric; otherwise it counts as REAL 0.0. Measured: `SUM(s)` over {'10', NULL, 'abc'} + is 10.0 REAL (so 'abc' contributed 0.0 and made the total real), not 10 INTEGER.""" + if isinstance(v, str): + n = apply_numeric_affinity(v) + return n if isinstance(n, (int, float)) else 0.0 + return v + + +def agg_count(values, n_rows): + """COUNT(expr): number of non-NULL values.""" + return sum(1 for v in values if v is not None) + + +def agg_sum(values): + """SUM: NULL when there is no non-NULL value; INTEGER when every value is an integer.""" + vals = [agg_numeric(v) for v in values if v is not None] + if not vals: + return None + if all(isinstance(v, int) for v in vals): + return sum(vals) + return float(sum(vals)) + + +def agg_avg(values): + vals = [agg_numeric(v) for v in values if v is not None] + if not vals: + return None + return float(sum(vals)) / len(vals) + + +def agg_minmax(values, want_max): + vals = [v for v in values if v is not None] + if not vals: + return None + best = vals[0] + for v in vals[1:]: + c = compare_values(v, best) + if (c > 0) if want_max else (c < 0): + best = v + return best + + +_LIKE_CACHE = {} + + +def like_match(value, pattern): + """SQLite LIKE: ASCII-case-insensitive, `%` = any run, `_` = exactly one character.""" + if value is None or pattern is None: + return None + if not isinstance(value, str): + value = str(value) + rx = _LIKE_CACHE.get(pattern) + if rx is None: + out = ["(?s)\\A"] + for ch in pattern: + if ch == "%": + out.append(".*") + elif ch == "_": + out.append(".") + else: + out.append(re.escape(ch)) + out.append("\\Z") + rx = re.compile("".join(out), re.IGNORECASE | re.ASCII) + _LIKE_CACHE[pattern] = rx + return 1 if rx.match(value) else 0 + + +def distinct_key(row): + """Equality for DISTINCT / GROUP BY: numbers unify (5 == 5.0), text and NULL by identity.""" + key = [] + for v in row: + if v is None: + key.append((0, None)) + elif isinstance(v, (int, float)) and not isinstance(v, bool): + key.append((1, float(v))) + elif isinstance(v, str): + key.append((2, v)) + else: + raise SqlError("unsupported value in DISTINCT: %r" % (type(v),)) + return tuple(key) + + +def join_relation(left, right, kind, on_eval): + """Nested-loop join. `kind` is 'inner' or 'left'; unmatched left rows get NULL right side.""" + out = [] + n_right_cols = len(right.cols) + for lrow in left.rows: + matched = False + for rrow in right.rows: + combined = lrow + rrow + if truthy(on_eval(combined)): + matched = True + out.append(combined) + if kind == "left" and not matched: + out.append(lrow + [None] * n_right_cols) + return Relation(left.cols + right.cols, out) + + +# -------------------------------------------------------------------------- +# relations / evaluation +# -------------------------------------------------------------------------- + +@dataclass +class Table: + cols: list # [(name, declared_type)] + rows: list # [[value, ...], ...] + + +@dataclass +class Relation: + cols: list # [(table_alias, column_name), ...] + rows: list # [] is used for the index-only scaffold built before a join + index: Optional[dict] = None # name -> [idx]; set by _build_index + + +@dataclass +class _Ctx: + rel: Relation + values: list + group: Optional[list] = None # list of value-lists when in an aggregate context + index: Optional[dict] = None # name -> [idx] built once per relation + + +def _build_index(rel): + idx = {} + for i, (tbl, col) in enumerate(rel.cols): + idx.setdefault(col, []).append(i) + idx.setdefault(tbl + "." + col, [i]) + return idx + + +def _resolve(ctx, tbl, col): + key = col if tbl is None else tbl + "." + col + hits = ctx.index.get(key) + if not hits: + raise SqlError("no such column: %s" % key) + if len(hits) > 1: + raise SqlError("ambiguous column: %s" % key) + return hits[0] + + +def affinity_of(node): + """Only a bare column reference carries affinity in this engine.""" + if node[0] == "col": + return node[3] if len(node) > 3 else None + return None + + +def has_aggregate(node): + k = node[0] + if k == "agg": + return True + for sub in node[1:]: + if isinstance(sub, tuple) and sub and isinstance(sub[0], str): + if has_aggregate(sub): + return True + elif isinstance(sub, list): + for x in sub: + if isinstance(x, tuple) and x and isinstance(x[0], str) and has_aggregate(x): + return True + if isinstance(x, tuple) and x and isinstance(x[0], tuple): + for y in x: + if isinstance(y, tuple) and y and isinstance(y[0], str) and has_aggregate(y): + return True + return False + + +def eval_expr(node, ctx): + k = node[0] + if k == "lit": + return node[1] + if k == "col": + tbl, col = node[1], node[2] + i = _resolve(ctx, tbl, col) + return ctx.values[i] + if k == "star": + raise SqlError("* is only valid in the select list") + if k == "unary": + v = eval_expr(node[2], ctx) + if v is None: + return None + if node[1] == "+": + return v + if isinstance(v, str): + raise SqlError("unary - on text is out of scope") + return -v + if k == "binop": + return arith(node[1], eval_expr(node[2], ctx), eval_expr(node[3], ctx)) + if k == "cmp": + return _eval_cmp(node, ctx) + if k == "isnull": + v = eval_expr(node[1], ctx) + return 1 if ((v is None) != bool(node[2])) else 0 + if k == "in": + return _eval_in(node, ctx) + if k == "between": + lo = eval_expr(node[2], ctx) + hi = eval_expr(node[3], ctx) + aff = affinity_of(node[1]) + ge = _cmp_result(eval_expr(node[1], ctx), lo, aff, affinity_of(node[2]), ">=") + le = _cmp_result(eval_expr(node[1], ctx), hi, aff, affinity_of(node[3]), "<=") + r = _tri_and(ge, le) + return _tri_not(r) if node[4] else r + if k == "like": + r = like_match(eval_expr(node[1], ctx), eval_expr(node[2], ctx)) + return _tri_not(r) if node[3] else r + if k == "and": + return _tri_and(eval_expr(node[1], ctx), eval_expr(node[2], ctx)) + if k == "or": + return _tri_or(eval_expr(node[1], ctx), eval_expr(node[2], ctx)) + if k == "not": + return _tri_not(eval_expr(node[1], ctx)) + if k == "case": + for cond, val in node[1]: + if truthy(eval_expr(cond, ctx)): + return eval_expr(val, ctx) + return None if node[2] is None else eval_expr(node[2], ctx) + if k == "agg": + return _eval_agg(node, ctx) + raise SqlError("cannot evaluate node %r" % (k,)) + + +def _cmp_result(a, b, aff_a, aff_b, op): + c = compare_values(a, b, aff_a, aff_b) + if c is None: + return None + if op == "=": + return 1 if c == 0 else 0 + if op in ("<>", "!="): + return 1 if c != 0 else 0 + if op == "<": + return 1 if c < 0 else 0 + if op == "<=": + return 1 if c <= 0 else 0 + if op == ">": + return 1 if c > 0 else 0 + if op == ">=": + return 1 if c >= 0 else 0 + raise SqlError("unknown comparison %r" % op) + + +def _eval_cmp(node, ctx): + return _cmp_result( + eval_expr(node[2], ctx), eval_expr(node[3], ctx), + affinity_of(node[2]), affinity_of(node[3]), node[1], + ) + + +def _eval_in(node, ctx): + """SQLite IN: a match is TRUE; otherwise NULL if any list element is NULL; else FALSE. + `NOT IN` is the three-valued negation of that, so `3 NOT IN (1,2,NULL)` is NULL.""" + target = eval_expr(node[1], ctx) + aff = affinity_of(node[1]) + saw_null = False + for item in node[2]: + v = eval_expr(item, ctx) + c = compare_values(target, v, aff, affinity_of(item)) + if c is None: + saw_null = True + elif c == 0: + return 0 if node[3] else 1 + r = None if saw_null else 0 + return _tri_not(r) if node[3] else r + + +def _tri_and(a, b): + ta, tb = tri(a), tri(b) + if ta == 0 or tb == 0: + return 0 + if ta is None or tb is None: + return None + return 1 + + +def _tri_or(a, b): + ta, tb = tri(a), tri(b) + if ta == 1 or tb == 1: + return 1 + if ta is None or tb is None: + return None + return 0 + + +def _tri_not(a): + ta = tri(a) + return None if ta is None else 1 - ta + + +def _eval_agg(node, ctx): + fname, arg = node[1], node[2] + if ctx.group is None: + raise SqlError("aggregate %s outside an aggregate context" % fname) + if fname == "COUNT" and arg is None: + return len(ctx.group) + values = [] + for rv in ctx.group: + values.append(eval_expr(arg, replace(ctx, values=rv, group=None))) + if fname == "COUNT": + return agg_count(values, len(ctx.group)) + if fname == "SUM": + return agg_sum(values) + if fname == "AVG": + return agg_avg(values) + if fname == "MIN": + return agg_minmax(values, False) + if fname == "MAX": + return agg_minmax(values, True) + raise SqlError("unknown aggregate %s" % fname) + + +# -------------------------------------------------------------------------- +# query execution +# -------------------------------------------------------------------------- + +def _build_from(q, tables): + tname, alias = q["from"] + if tname is None: + return Relation([], [[]]) # SELECT without FROM: exactly one empty row + if tname not in tables: + raise SqlError("no such table: %s" % tname) + decl = {c: ty for t in tables.values() for c, ty in t.cols} + t = tables[tname] + rel = Relation([(alias, c) for c, _ in t.cols], [list(r) for r in t.rows]) + rel.index = _build_index(rel) + for kind, jtname, jalias, on in q["joins"]: + if jtname not in tables: + raise SqlError("no such table: %s" % jtname) + jt = tables[jtname] + right = Relation([(jalias, c) for c, _ in jt.cols], [list(r) for r in jt.rows]) + right.index = _build_index(right) + combined_cols = rel.cols + right.cols + combined_index = _build_index(Relation(combined_cols, [])) + # the ON clause gets the declared affinities too: `text_col <> 20` means `<> '20'` + col_aff = {i: decl.get(c) for i, (_a, c) in enumerate(combined_cols)} + + def on_eval(combined, _on=on, _cols=combined_cols, _idx=combined_index, _aff=col_aff): + ctx = _Ctx(Relation(_cols, []), combined, None, _idx) + return eval_expr(_tag(_on, ctx, _aff), ctx) + + rel = join_relation(rel, right, kind, on_eval) + rel.index = _build_index(rel) + return rel + + +def execute(sql_or_query, tables): + """Run a query against {name: Table} and return (column_names, rows).""" + q = sql_or_query if isinstance(sql_or_query, dict) else parse_query(sql_or_query) + rel = _build_from(q, tables) + + # affinity lookup for column refs (position -> declared type) + decl = {} + for t in tables.values(): + for cname, ctype in t.cols: + decl[cname] = ctype + col_aff = {i: decl.get(c) for i, (_a, c) in enumerate(rel.cols)} + + rows = rel.rows + if q["where"] is not None: + kept = [] + for rv in rows: + ctx = _Ctx(rel, rv, None, rel.index) + if truthy(_eval_with_affinity(q["where"], ctx, col_aff)): + kept.append(rv) + rows = kept + + agg_context = bool(q["group_by"]) \ + or any(it[0] == "item" and has_aggregate(it[1]) for it in q["items"]) \ + or (q["having"] is not None and has_aggregate(q["having"])) \ + or any(has_aggregate(e) for e, _ in q["order_by"]) + + out_names, out_exprs = _expand_items(q["items"], rel) + + # SQLite resolves a select-list alias inside GROUP BY / HAVING / ORDER BY; substitute it + # with the aliased expression so `HAVING n > 1` means `HAVING COUNT(*) > 1`. + alias_expr = {} + for (nm, _e), e in zip(zip(out_names, out_exprs), out_exprs): + if nm: + alias_expr.setdefault(nm, e) + + def sub_alias(node): + if node[0] == "col" and node[1] is None and node[2] in alias_expr: + return alias_expr[node[2]] + out = [node[0]] + for s in node[1:]: + if isinstance(s, tuple) and s and isinstance(s[0], str): + out.append(sub_alias(s)) + elif isinstance(s, list): + out.append([sub_alias(x) if (isinstance(x, tuple) and x and isinstance(x[0], str)) + else x for x in s]) + else: + out.append(s) + return tuple(out) + + group_by = [sub_alias(e) for e in q["group_by"]] + having = sub_alias(q["having"]) if q["having"] is not None else None + + produced = [] # (values, ctx) + if agg_context: + if group_by: + groups = [] + seen = {} + for rv in rows: + ctx = _Ctx(rel, rv, None, rel.index) + key = distinct_key([_eval_with_affinity(e, ctx, col_aff) for e in group_by]) + if key in seen: + groups[seen[key]][1].append(rv) + else: + seen[key] = len(groups) + groups.append((rv, [rv])) + else: + # a bare aggregate over zero rows still yields exactly one row (COUNT 0 / SUM NULL) + groups = [(rows[0] if rows else [], rows)] + for first, grp in groups: + ctx = _Ctx(rel, first, grp, rel.index) + if having is not None and not truthy(_eval_with_affinity(having, ctx, col_aff)): + continue + vals = [_eval_with_affinity(e, ctx, col_aff) for e in out_exprs] + produced.append((vals, ctx)) + else: + for rv in rows: + ctx = _Ctx(rel, rv, None, rel.index) + vals = [_eval_with_affinity(e, ctx, col_aff) for e in out_exprs] + produced.append((vals, ctx)) + + if q["distinct"]: + seen, uniq = set(), [] + for vals, ctx in produced: + k = distinct_key(vals) + if k not in seen: + seen.add(k) + uniq.append((vals, ctx)) + produced = uniq + + if q["order_by"]: + alias_pos = {} + for i, (nm, _e) in enumerate(zip(out_names, out_exprs)): + if nm: + alias_pos.setdefault(nm, i) + keys = [] + for vals, ctx in produced: + row_key = [] + for e, desc in q["order_by"]: + e = sub_alias(e) + if e[0] == "col" and e[1] is None and e[2] in alias_pos: + v = vals[alias_pos[e[2]]] + else: + v = _eval_with_affinity(e, ctx, col_aff) + row_key.append((sort_key(v), desc)) + keys.append(row_key) + order = sorted(range(len(produced)), + key=lambda i: _order_tuple(keys[i])) + produced = [produced[i] for i in order] + + if q["limit"] is not None: + start = q["offset"] or 0 + produced = produced[start:start + q["limit"]] if q["limit"] >= 0 else produced[start:] + elif q["offset"]: + produced = produced[q["offset"]:] + + return out_names, [v for v, _ in produced] + + +class _Desc: + __slots__ = ("v",) + + def __init__(self, v): + self.v = v + + def __lt__(self, other): + return other.v < self.v + + def __eq__(self, other): + return other.v == self.v + + +def _order_tuple(row_key): + return tuple(_Desc(k) if desc else k for k, desc in row_key) + + +def _eval_with_affinity(node, ctx, col_aff): + return eval_expr(_tag(node, ctx, col_aff), ctx) + + +def _tag(node, ctx, col_aff): + """Recursively attach declared affinities to bare column refs (see affinity_of).""" + k = node[0] + if k == "col": + i = _resolve(ctx, node[1], node[2]) + return ("col", node[1], node[2], col_aff.get(i)) + if k == "agg": + return node + out = [k] + for sub in node[1:]: + if isinstance(sub, tuple) and sub and isinstance(sub[0], str): + out.append(_tag(sub, ctx, col_aff)) + elif isinstance(sub, list): + new = [] + for x in sub: + if isinstance(x, tuple) and x and isinstance(x[0], str): + new.append(_tag(x, ctx, col_aff)) + elif isinstance(x, tuple) and x and isinstance(x[0], tuple): + new.append(tuple(_tag(y, ctx, col_aff) if isinstance(y, tuple) + else y for y in x)) + else: + new.append(x) + out.append(new) + else: + out.append(sub) + return tuple(out) + + +def _expand_items(items, rel): + names, exprs = [], [] + for it in items: + if it[0] == "star": + tbl = it[1] + for (_alias, col) in rel.cols: + if tbl is None or _alias == tbl: + names.append(col) + exprs.append(("col", _alias, col)) + continue + e, alias = it[1], it[2] + if alias: + names.append(alias) + elif e[0] == "col": + names.append(e[2]) + else: + names.append(None) + exprs.append(e) + return names, exprs + + +def run(sql, tables): + """Convenience: execute SQL text against {name: Table}.""" + return execute(sql, tables) + + +def table_from(cols, rows): + return Table([(c, t) for c, t in cols], [list(r) for r in rows]) + + +# -------------------------------------------------------------------------- +# selftest -- the differential harness in diff_sqlite.py is the real teeth +# -------------------------------------------------------------------------- + +_SELFTEST = [ + ("SELECT 1 + 2 * 3", [], [[7]]), + ("SELECT 7 / 2", [], [[3]]), + ("SELECT -7 / 2", [], [[-3]]), + ("SELECT 7.0 / 2", [], [[3.5]]), + ("SELECT 1 / 0", [], [[None]]), + ("SELECT NULL = NULL", [], [[None]]), + ("SELECT 1 IS NULL", [], [[0]]), + ("SELECT NULL IS NULL", [], [[1]]), + ("SELECT 'a' < 5", [], [[0]]), + ("SELECT 5 < 'a'", [], [[1]]), + ("SELECT '5abc' + 0", [], [[5]]), + ("SELECT 'a' + 0", [], [[0]]), + ("SELECT '5.5' + 0", [], [[5.5]]), + ("SELECT 'abc' LIKE 'A%'", [], [[1]]), + ("SELECT 'abc' LIKE 'a_c'", [], [[1]]), + ("SELECT 'abc' NOT LIKE 'b%'", [], [[1]]), + ("SELECT 2 IN (1, 2, NULL)", [], [[1]]), + ("SELECT 3 IN (1, 2, NULL)", [], [[None]]), + ("SELECT 3 NOT IN (1, 2, NULL)", [], [[None]]), + ("SELECT 3 NOT IN (1, 2)", [], [[1]]), + ("SELECT 2 BETWEEN 1 AND 3", [], [[1]]), + ("SELECT CASE WHEN 1 = 2 THEN 'a' ELSE 'b' END", [], [['b']]), +] + + +def _selftest(): + tables = { + "t": table_from([("id", "INTEGER"), ("s", "TEXT")], [[1, "a"], [2, None], [3, "c"]]), + } + fails = 0 + for sql, _t, want in _SELFTEST: + got = execute(sql, tables)[1] + if got != want: + fails += 1 + print("SELFTEST FAIL: %s -> %r (want %r)" % (sql, got, want)) + extra = [ + ("SELECT COUNT(*), COUNT(s), SUM(id) FROM t", [[3, 2, 6]]), + ("SELECT SUM(s) FROM t", [[0.0]]), # text coerces to REAL 0.0 + ("SELECT SUM(id) FROM t WHERE id > 99", [[None]]), + ("SELECT * FROM t ORDER BY s", [[2, None], [1, "a"], [3, "c"]]), + ("SELECT s, COUNT(*) AS n FROM t GROUP BY s ORDER BY n, s", + [[None, 1], ["a", 1], ["c", 1]]), + ("SELECT s, COUNT(*) AS n FROM t GROUP BY s HAVING n > 1", []), + ("SELECT id FROM t WHERE s IS NULL", [[2]]), + ("SELECT id FROM t ORDER BY id DESC LIMIT 2", [[3], [2]]), + ("SELECT id FROM t ORDER BY id LIMIT 2 OFFSET 1", [[2], [3]]), + ("SELECT id FROM t ORDER BY id LIMIT -1 OFFSET 1", [[2], [3]]), + ] + for sql, want in extra: + try: + got = execute(sql, tables)[1] + except SqlError as exc: + got = "SqlError: %s" % exc + if got != want: + fails += 1 + print("SELFTEST FAIL: %s -> %r (want %r)" % (sql, got, want)) + print("selftest: %d/%d" % (len(_SELFTEST) + len(extra) - fails, len(_SELFTEST) + len(extra))) + return 0 if fails == 0 else 1 + + +if __name__ == "__main__": + if "--selftest" in sys.argv: + sys.exit(_selftest()) + print(__doc__.strip().splitlines()[1]) + print("usage: minisql.py --selftest (the differential battery is diff_sqlite.py)") diff --git a/wizard-work/null/raglite/README.md b/wizard-work/null/raglite/README.md new file mode 100644 index 0000000..fa697b7 --- /dev/null +++ b/wizard-work/null/raglite/README.md @@ -0,0 +1,46 @@ +# raglite — inverted index + BM25 retrieval (stdlib only) + +Minimal passage retrieval for agents: index a file or directory of markdown/text, +rank with BM25, get JSON hits. No dependencies, no network, deterministic. + +## Files + +- `raglite.py` — library + CLI +- `test_raglite.py` — self-contained test suite (own fixtures, own temp dirs) + +## Library + +```python +import raglite +idx = raglite.build_index(["./docs"]) # file or directory; recursive +raglite.save_index(idx, "index.json") +idx = raglite.load_index("index.json") +hits = raglite.search(idx, "bm25 ranking", k=3) +# -> [{"id": "docs/bm25.md::0", "score": 3.63..., "text": "..."}] +``` + +A passage is a blank-line-delimited block, addressed `::`. +The index is plain JSON (`postings`, `lengths`, `texts`, `n_docs`, `avgdl`, `k1`, `b`). + +## CLI + +``` +python3 raglite.py index ./docs --out index.json +python3 raglite.py query index.json "bm25 ranking" -k 3 +python3 raglite.py query index.json "bm25 ranking" -k 3 --json +``` + +## Evidence (measured, 2026-10-22) + +`python3 test_raglite.py` → **25/25 checks pass, exit 0**. + +Checks cover: tokenizer punctuation/lowercasing, passage splitting, passage count +(4 docs × 2 paragraphs = 8), top-hit correctness for three queries, unknown-term → +no hits, positive and descending scores, a negative control (codeforge must not rank +for a kibana query), JSON round-trip identity, exact BM25 constants on a 2-doc corpus +(compared to values recomputed independently in `bm25_probe.py`), tf ordering, and the +CLI end-to-end via subprocess (exit codes, passage count, valid JSON, top hit). + +Real-corpus smoke test: `raglite.py index /root/troupe-standup/wizard-work/null` +→ **139 passages**; query `"json patch apply"` → top hit `jsonpatch/README.md::0`; +query `"scheduler cron"` → top hit `schedkit/README.md::0`. diff --git a/wizard-work/null/raglite/raglite.py b/wizard-work/null/raglite/raglite.py new file mode 100644 index 0000000..e319260 --- /dev/null +++ b/wizard-work/null/raglite/raglite.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""raglite -- minimal retrieval for agents: inverted index + BM25 over passages. + +Stdlib only. Deterministic. No network. No dependencies. + +A "passage" is a blank-line-delimited block of a text/markdown file, addressed +as "::". The index is plain JSON so it can be diffed, +cached and inspected by hand. + +Library: + idx = raglite.build_index(["./docs"]) # path may be a file or a directory + raglite.save_index(idx, "index.json") + idx = raglite.load_index("index.json") + hits = raglite.search(idx, "bm25 ranking", k=3) + +CLI: + python3 raglite.py index ./docs --out index.json + python3 raglite.py query index.json "bm25 ranking" -k 3 --json +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import sys +from collections import Counter, defaultdict +from pathlib import Path + +TOKEN_RE = re.compile(r"[A-Za-z0-9_']+") +PASSAGE_SPLIT = re.compile(r"\n[ \t]*\n") +DEFAULT_EXTENSIONS = (".md", ".txt") + + +def tokenize(text: str) -> list[str]: + """Lowercased alphanumeric tokens; punctuation is a separator.""" + return TOKEN_RE.findall(text.lower()) + + +def split_passages(text: str) -> list[str]: + """Split on blank lines, drop empties, keep original order.""" + return [p.strip() for p in PASSAGE_SPLIT.split(text) if p.strip()] + + +def build_index(paths, k1: float = 1.5, b: float = 0.75, + extensions=DEFAULT_EXTENSIONS) -> dict: + """Build a BM25 index over every passage of every file under `paths`.""" + postings: dict[str, Counter] = defaultdict(Counter) + lengths: dict[str, int] = {} + texts: dict[str, str] = {} + n = 0 + + for raw in paths: + p = Path(raw) + if p.is_dir(): + files = sorted(q for q in p.rglob("*") + if q.suffix.lower() in extensions) + elif p.is_file(): + files = [p] + else: + raise FileNotFoundError(f"no such path: {raw}") + + for f in files: + body = f.read_text(encoding="utf-8", errors="replace") + for i, passage in enumerate(split_passages(body)): + toks = tokenize(passage) + if not toks: + continue + pid = f"{f}::{i}" + for term, tf in Counter(toks).items(): + postings[term][pid] = tf + lengths[pid] = len(toks) + texts[pid] = passage + n += 1 + + total = sum(lengths.values()) + return { + "k1": k1, + "b": b, + "n_docs": n, + "avgdl": (total / n) if n else 0.0, + "lengths": lengths, + "texts": texts, + "postings": {t: dict(c) for t, c in postings.items()}, + } + + +def _idf(df: int, n: int) -> float: + """BM25 idf, Robertson's non-negative form.""" + return math.log(1.0 + (n - df + 0.5) / (df + 0.5)) + + +def search(idx: dict, query: str, k: int = 5) -> list[dict]: + """Rank passages for `query`; returns <= k hits, best first. + + Each hit: {"id", "score", "text"}. Score is the raw BM25 sum (unrounded). + """ + k1, b = idx["k1"], idx["b"] + n, avgdl = idx["n_docs"], idx["avgdl"] + if not avgdl or k <= 0: + return [] + + scores: dict[str, float] = defaultdict(float) + for term in set(tokenize(query)): + post = idx["postings"].get(term) + if not post: + continue + idf = _idf(len(post), n) + for pid, tf in post.items(): + dl = idx["lengths"][pid] + denom = tf + k1 * (1.0 - b + b * dl / avgdl) + scores[pid] += idf * (tf * (k1 + 1.0)) / denom + + ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0])) + return [{"id": pid, "score": s, "text": idx["texts"][pid]} + for pid, s in ranked[:k]] + + +def save_index(idx: dict, path) -> None: + Path(path).write_text(json.dumps(idx, sort_keys=True), encoding="utf-8") + + +def load_index(path) -> dict: + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(prog="raglite", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + + pi = sub.add_parser("index", help="build an index over a file or directory") + pi.add_argument("path") + pi.add_argument("--out", required=True) + pi.add_argument("--k1", type=float, default=1.5) + pi.add_argument("--b", type=float, default=0.75) + + pq = sub.add_parser("query", help="rank passages in an index") + pq.add_argument("index") + pq.add_argument("query") + pq.add_argument("-k", type=int, default=5) + pq.add_argument("--json", action="store_true", help="emit JSON hits") + + args = ap.parse_args(argv) + + if args.cmd == "index": + idx = build_index([args.path], k1=args.k1, b=args.b) + save_index(idx, args.out) + print(f"indexed {idx['n_docs']} passages from {args.path} -> {args.out}") + return 0 + + idx = load_index(args.index) + hits = search(idx, args.query, k=args.k) + if args.json: + print(json.dumps(hits, indent=2)) + else: + for i, h in enumerate(hits, 1): + print(f"{i}. [{h['score']:.6f}] {h['id']}") + print(f" {h['text'][:160]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/null/raglite/test_raglite.py b/wizard-work/null/raglite/test_raglite.py new file mode 100644 index 0000000..2968afd --- /dev/null +++ b/wizard-work/null/raglite/test_raglite.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Self-contained tests for raglite.py. + +Creates its own fixtures in temp dirs, exercises the library and the CLI, +and exits 0 only if every check passes. No network, no external deps. + +Run: python3 /root/.hermes/wizard-outputs/null/test_raglite.py +""" + +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import raglite # noqa: E402 + +FAILURES = [] + + +def check(name, cond, detail=""): + if cond: + print(f" ok {name}") + else: + print(f" FAIL {name} {detail}") + FAILURES.append(name) + + +def doc_of(pid: str) -> str: + """'path/to/codeforge.md::3' -> 'codeforge.md'""" + return pid.split("::")[0].rsplit("/", 1)[-1] + + +DOCS = { + "codeforge.md": ( + "The wizard forged the spell in the codeforge.\n" + "\n" + "The codeforge hums with runes and ember-light.\n" + ), + "kibana.md": ( + "Kibana dashboards visualize logs from the elasticsearch cluster.\n" + "\n" + "Operators drill into slow queries here.\n" + ), + "bm25.md": ( + "BM25 ranks passages by term frequency and inverse document frequency.\n" + "\n" + "It is a bag-of-words retrieval function.\n" + ), + "standup.md": ( + "The troupe stands up at dawn.\n" + "\n" + "Each wizard reports their lane and blockers.\n" + ), +} + + +def write_corpus(root): + for name, body in DOCS.items(): + (root / name).write_text(body, encoding="utf-8") + + +def test_tokenize(): + print("[tokenize]") + toks = raglite.tokenize("Hello, WORLD! It's a BM25-test.") + check("punctuation stripped, lowercased", + toks == ["hello", "world", "it's", "a", "bm25", "test"], repr(toks)) + check("empty input -> empty list", raglite.tokenize("") == []) + + +def test_split_passages(): + print("[split_passages]") + parts = raglite.split_passages("a\n\nb\n\n\nc\n") + check("blank-line split", parts == ["a", "b", "c"], repr(parts)) + check("whitespace-only block dropped", raglite.split_passages("a\n \nb") == ["a", "b"]) + + +def test_index_and_ranking(tmp): + print("[index + ranking]") + idx = raglite.build_index([str(tmp)]) + check("4 docs x 2 paragraphs -> 8 passages", idx["n_docs"] == 8, idx["n_docs"]) + check("avgdl is positive", idx["avgdl"] > 0, idx["avgdl"]) + + h = raglite.search(idx, "codeforge", k=3) + check("query 'codeforge' returns hits", len(h) > 0) + check("top hit is the codeforge passage", + bool(h) and doc_of(h[0]["id"]) == "codeforge.md" + and "codeforge" in h[0]["text"], + h[0]["id"] if h else "none") + + h = raglite.search(idx, "elasticsearch logs", k=3) + check("top hit for 'elasticsearch logs' is kibana", + bool(h) and doc_of(h[0]["id"]) == "kibana.md", h[0]["id"] if h else "none") + + h = raglite.search(idx, "bm25 ranking", k=3) + check("top hit for 'bm25 ranking' is bm25.md", + bool(h) and doc_of(h[0]["id"]) == "bm25.md", h[0]["id"] if h else "none") + + h = raglite.search(idx, "nonexistentterm", k=5) + check("unknown term -> no hits", h == [], repr(h)) + + h = raglite.search(idx, "the", k=5) + check("scores strictly positive", all(x["score"] > 0 for x in h), repr(h)) + check("scores sorted descending", + all(h[i]["score"] >= h[i + 1]["score"] for i in range(len(h) - 1)), repr(h)) + + # Negative control: a kibana query must not surface the codeforge doc. + h = raglite.search(idx, "kibana dashboards", k=5) + check("negative control: codeforge not ranked for kibana query", + all(doc_of(x["id"]) != "codeforge.md" for x in h), + repr([x["id"] for x in h])) + + +def test_roundtrip(tmp): + print("[json round-trip]") + idx = raglite.build_index([str(tmp)]) + path = tmp / "index.json" + raglite.save_index(idx, path) + check("index file written", path.exists() and path.stat().st_size > 0) + + loaded = raglite.load_index(path) + a = raglite.search(idx, "codeforge", k=5) + b = raglite.search(loaded, "codeforge", k=5) + check("loaded index gives identical hits", + [(x["id"], x["score"]) for x in a] == [(x["id"], x["score"]) for x in b], + f"{a} != {b}") + + +def test_exact_bm25(): + """Compare against BM25 values computed independently (bm25_probe.py).""" + print("[exact bm25 constants]") + with tempfile.TemporaryDirectory(prefix="raglite_tiny_") as d: + tiny = Path(d) + (tiny / "d1.md").write_text("a a b\n", encoding="utf-8") + (tiny / "d2.md").write_text("a c\n", encoding="utf-8") + idx = raglite.build_index([str(tiny)]) + + # Independently recomputed with the same Robertson formula (bm25_probe.py). + EXPECTED = { + ("d1.md", "a"): 0.24472692187108, + ("d2.md", "a"): 0.20035335911423582, + ("d1.md", "b"): 0.6359148445504086, + ("d2.md", "c"): 0.7617001984175223, + } + for (doc, term), want in EXPECTED.items(): + hits = raglite.search(idx, term, k=10) + got = next((x["score"] for x in hits if doc_of(x["id"]) == doc), None) + check(f"score[{doc},{term}] == {want:.12f}", + got is not None and abs(got - want) < 1e-12, + f"got {got!r}") + + # Higher term frequency must win for the same term. + hits = raglite.search(idx, "a", k=2) + check("tf ordering: d1 (tf=2) ranks above d2 (tf=1)", + len(hits) == 2 and doc_of(hits[0]["id"]) == "d1.md", repr(hits)) + + +def test_cli(tmp): + print("[cli]") + out = tmp / "cli_index.json" + r = subprocess.run([sys.executable, str(HERE / "raglite.py"), + "index", str(tmp), "--out", str(out)], + capture_output=True, text=True) + check("cli index exits 0", r.returncode == 0, r.stderr[-400:]) + check("cli index reports 8 passages", "8 passages" in r.stdout, r.stdout.strip()) + + r = subprocess.run([sys.executable, str(HERE / "raglite.py"), + "query", str(out), "elasticsearch logs", "-k", "2", "--json"], + capture_output=True, text=True) + check("cli query exits 0", r.returncode == 0, r.stderr[-400:]) + try: + hits = json.loads(r.stdout) + except json.JSONDecodeError as e: + check("cli query emits valid JSON", False, f"{e}: {r.stdout[:200]}") + return + check("cli query emits valid JSON", True) + check("cli top hit is kibana", + bool(hits) and doc_of(hits[0]["id"]) == "kibana.md", repr(hits)) + + +def main(): + print("raglite test suite") + test_tokenize() + test_split_passages() + with tempfile.TemporaryDirectory(prefix="raglite_test_") as d: + tmp = Path(d) + write_corpus(tmp) + test_index_and_ranking(tmp) + test_roundtrip(tmp) + test_cli(tmp) + test_exact_bm25() + print() + if FAILURES: + print(f"FAILED ({len(FAILURES)}): " + ", ".join(FAILURES)) + return 1 + print("ALL CHECKS PASSED") + return 0 + + +if __name__ == "__main__": + sys.exit(main())