#!/usr/bin/env python3 """ diff_sqlite.py -- randomized differential battery: minisql.py vs sqlite3 (the oracle). Wizard: null (lane: Codeforge -- hard coding and agentic reasoning). What it does, per query: 1. generates a random schema (2-3 tables, INTEGER/TEXT/REAL columns, NULLs, 0-12 rows), 2. generates one random SELECT in the supported grammar, 3. creates the SAME schema in an in-memory sqlite3 database and executes the SAME SQL text in both engines, 4. compares the result sets -- strictly, by type category (int / real / text / null) and by value -- as an ordered sequence when the query has ORDER BY, else as a multiset. It exits 0 only when ALL of these hold, so the exit code cannot pass vacuously: * sqlite3 is importable and every generated statement ran on BOTH engines without error, * the requested number of queries actually executed, * every query shape was exercised (const / scan / where / join / group / agg / distinct), * at least a set fraction of queries returned at least one row, and the battery compared at least a set number of rows (so agreement is not agreement about empty sets), * zero mismatches, * with --mutants: every seeded engine defect is CAUGHT (each mutant must produce at least one mismatch). A mutant that survives means this battery has no teeth for that semantics. Generator constraints (each one is there to keep the oracle deterministic, and each is a documented hole rather than an accident): * every ORDER BY is made total by appending the `id` columns of the FROM tables, so ties cannot reorder differently in the two engines; * DISTINCT queries are never ordered (SQLite requires ORDER BY terms to be in the result set, and ordered ties would not be comparable) -- they are compared as multisets; * LIMIT/OFFSET only appear with an ORDER BY; * with GROUP BY, the select list is group keys plus aggregates only -- never a bare column of a multi-row group, whose value SQLite leaves unspecified. Usage: python3 diff_sqlite.py [--queries N] [--seed S] [--mutants] [--max-print K] [--quiet] """ from __future__ import annotations import argparse import os import random import sqlite3 import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import minisql # noqa: E402 SHAPES = ("const", "scan", "where", "join", "group", "agg", "distinct") TEXT_POOL = ["apple", "banana", "cherry", "a", "ab", "", "A", "10", "20"] LIKE_POOL = ["%a%", "a%", "%b", "_", "ab", "A%", "%", "", "c_erry", "ap_le", "%rry"] NUM_LITERALS = ["0", "1", "-1", "2", "5", "10", "20", "3.5", "5.5", "0.0"] STR_NUM_LITERALS = ["0", "5", "10", "20", "5.5", "abc", ""] # -------------------------------------------------------------------------- # schema # -------------------------------------------------------------------------- def make_schema(rng): """Return (spec, rows): spec is {table: [(col, declared_type)]}, rows is {table: [[v...]]}. `id` is a unique non-NULL INTEGER in every table; it is the tiebreaker the generator appends to every ORDER BY, so ordered results are deterministic in both engines. """ spec = { "t1": [("id", "INTEGER"), ("a", "INTEGER"), ("b", "TEXT"), ("c", "REAL")], "t2": [("id", "INTEGER"), ("d", "INTEGER"), ("e", "TEXT")], } if rng.random() < 0.5: spec["t3"] = [("id", "INTEGER"), ("f", "REAL"), ("g", "TEXT")] rows = {} for name, cols in spec.items(): n = rng.randint(0, 12) table_rows = [] for i in range(1, n + 1): row = [] for cname, ctype in cols: if cname == "id": row.append(i) elif rng.random() < 0.2: row.append(None) elif ctype == "INTEGER": row.append(rng.randint(0, 20)) elif ctype == "REAL": row.append(round(rng.uniform(0, 10), 1)) else: row.append(rng.choice(TEXT_POOL)) table_rows.append(row) rows[name] = table_rows return spec, rows def load_sqlite(con, spec, rows): for name, cols in spec.items(): con.execute("CREATE TABLE %s (%s)" % ( name, ", ".join("%s %s" % (c, t) for c, t in cols))) if rows[name]: con.executemany("INSERT INTO %s VALUES (%s)" % ( name, ", ".join("?" * len(cols))), rows[name]) # -------------------------------------------------------------------------- # query generator # -------------------------------------------------------------------------- class Gen: """Emits one random SELECT. All column references are drawn from the tables named in `scope`, which the query method sets to exactly the FROM tables before generating.""" def __init__(self, rng, spec, rows): self.rng = rng self.spec = spec self.rows = rows self.tables = list(spec) self.scope = [] self.pool_num = [] # [(table, col)] with INTEGER/REAL type, inside scope self.pool_text = [] # [(table, col)] with TEXT type, inside scope self.pool_all = [] # [(table, col)] any type except id self.pool_id = [] # [(table, "id")] # -- scope ------------------------------------------------------------ def set_scope(self, tables): self.scope = list(tables) self.pool_num, self.pool_text, self.pool_all, self.pool_id = [], [], [], [] for t in self.scope: for c, ty in self.spec[t]: if c == "id": self.pool_id.append((t, c)) else: self.pool_all.append((t, c)) (self.pool_num if ty in ("INTEGER", "REAL") else self.pool_text).append((t, c)) def ref(self, tc, qualify): t, c = tc return "%s.%s" % (t, c) if qualify else c def pick(self, pool): return self.rng.choice(pool) if pool else None # -- literals --------------------------------------------------------- def num_literal(self): return self.rng.choice(NUM_LITERALS) def text_literal(self): return "'%s'" % self.rng.choice(TEXT_POOL) # -- expressions ------------------------------------------------------ def numeric_expr(self, qualify, depth=0): pool = self.pool_num r = self.rng.random() if not pool or r < 0.4 or depth >= 2: return self.ref(self.pick(pool), qualify) if pool else self.num_literal() if r < 0.55: return self.num_literal() if r < 0.68: return "(%s + %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal()) if r < 0.8: return "(%s * %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal()) if r < 0.9: return "(%s - %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal()) # division is here on purpose: SQLite's `/` stays INTEGER for two integers, and the # `int-div-becomes-real` mutant exists to prove this battery notices when it does not. return "(%s / %s)" % (self.numeric_expr(qualify, depth + 1), self.rng.choice(["2", "3", "1", "4"])) def scalar_expr(self, qualify, depth=0): r = self.rng.random() if r < 0.45: return self.numeric_expr(qualify, depth) if r < 0.6 and self.pool_text: return self.ref(self.pick(self.pool_text), qualify) if r < 0.7: return self.text_literal() if r < 0.8 and depth < 2: return "CASE WHEN %s THEN %s ELSE %s END" % ( self.predicate(qualify, depth + 1), self.scalar_expr(qualify, depth + 1), self.scalar_expr(qualify, depth + 1)) return self.predicate(qualify, depth + 1) def predicate(self, qualify, depth=0): r = self.rng.random() if depth >= 2: r = min(r, 0.59) if r < 0.35: return self.comparison(qualify) if r < 0.45: return "(%s AND %s)" % (self.predicate(qualify, depth + 1), self.predicate(qualify, depth + 1)) if r < 0.55: return "(%s OR %s)" % (self.predicate(qualify, depth + 1), self.predicate(qualify, depth + 1)) if r < 0.6: return "NOT %s" % self.predicate(qualify, depth + 1) if r < 0.68 and self.pool_all: tc = self.pick(self.pool_all) return "%s IS%s NULL" % (self.ref(tc, qualify), "" if self.rng.random() < 0.5 else " NOT") if r < 0.82 and self.pool_num: items = ", ".join(self.num_literal() for _ in range(self.rng.randint(1, 3))) return "%s%s IN (%s)" % (self.ref(self.pick(self.pool_num), qualify), " NOT" if self.rng.random() < 0.3 else "", items) if r < 0.88 and self.pool_text: items = ", ".join(self.text_literal() for _ in range(self.rng.randint(1, 3))) return "%s%s IN (%s)" % (self.ref(self.pick(self.pool_text), qualify), " NOT" if self.rng.random() < 0.3 else "", items) if r < 0.93 and self.pool_num: return "%s%s BETWEEN %s AND %s" % ( self.ref(self.pick(self.pool_num), qualify), " NOT" if self.rng.random() < 0.3 else "", self.num_literal(), self.num_literal()) if self.pool_text: return "%s%s LIKE '%s'" % (self.ref(self.pick(self.pool_text), qualify), " NOT" if self.rng.random() < 0.3 else "", self.rng.choice(LIKE_POOL)) return self.comparison(qualify) def comparison(self, qualify): """The mixed-affinity forms are deliberate: they are where an engine that ignores SQLite's two affinity rules diverges from it.""" r = self.rng.random() op = self.rng.choice(["=", "<>", "<", "<=", ">", ">="]) if r < 0.35 and self.pool_num: return "%s %s %s" % (self.ref(self.pick(self.pool_num), qualify), op, self.num_literal()) if r < 0.55 and self.pool_text: return "%s %s %s" % (self.ref(self.pick(self.pool_text), qualify), op, self.text_literal()) if r < 0.7 and self.pool_num: # numeric affinity is applied to the text literal return "%s %s '%s'" % (self.ref(self.pick(self.pool_num), qualify), op, self.rng.choice(STR_NUM_LITERALS)) if r < 0.85 and self.pool_text: # TEXT affinity is applied to the numeric literal return "%s %s %s" % (self.ref(self.pick(self.pool_text), qualify), op, self.rng.choice(["0", "5", "10", "20", "-1"])) if self.pool_num: return "%s %s %s" % (self.numeric_expr(qualify, 1), op, self.num_literal()) return self.comparison(qualify) if (self.pool_text or self.pool_num) else "1 = 1" # -- select items ----------------------------------------------------- def agg_item(self, qualify, i): fn = self.rng.choice(["COUNT", "SUM", "AVG", "MIN", "MAX"]) if fn == "COUNT" and self.rng.random() < 0.4: return "COUNT(*) AS agg%d" % i pool = self.pool_num if (fn in ("SUM", "AVG") and self.pool_num) else self.pool_all if not pool: return "COUNT(*) AS agg%d" % i return "%s(%s) AS agg%d" % (fn, self.ref(self.pick(pool), qualify), i) def items(self, qualify, agg=False, group_keys=None): out = [] if not agg and self.rng.random() < 0.25: out.append("*") for i in range(self.rng.randint(1, 3)): if agg: if group_keys and self.rng.random() < 0.35: out.append("%s AS gk%d" % (self.rng.choice(group_keys), i)) else: out.append(self.agg_item(qualify, i)) else: e = self.scalar_expr(qualify) out.append("%s AS x%d" % (e, i) if self.rng.random() < 0.6 else e) if not out: out.append("*") return out def having(self, gkeys): r = self.rng.random() if r < 0.45: return "COUNT(*) > %d" % self.rng.randint(0, 3) if r < 0.6 and self.pool_num: return "SUM(%s) > %s" % (self.ref(self.pick(self.pool_num), False), self.rng.choice(["5", "10", "20", "0"])) if r < 0.7 and self.pool_num: return "AVG(%s) > %s" % (self.ref(self.pick(self.pool_num), False), self.rng.choice(["1", "5", "10"])) if r < 0.8 and self.pool_all: return "MAX(%s) > %s" % (self.ref(self.pick(self.pool_all), False), self.rng.choice(["5", "10", "1.5"])) if r < 0.9 and gkeys: return "%s IS NOT NULL" % self.rng.choice(gkeys) return "COUNT(*) > 1" def order_key(self, qualify): """ORDER BY terms are always column references: a bare integer literal in ORDER BY is an ORDINAL in SQLite, not a constant, and the oracle rejects it out of range.""" pool = list(self.pool_all) + list(self.pool_id) if not pool: return "1 = 1" return self.ref(self.rng.choice(pool), qualify) def order_tail(self, keys, tiebreak): """keys: ordering exprs; tiebreak: exprs that make the order total.""" terms = ["%s%s" % (k, " DESC" if self.rng.random() < 0.4 else "") for k in keys] terms.extend(tiebreak) tail = " ORDER BY " + ", ".join(terms) if self.rng.random() < 0.35: tail += " LIMIT %d" % self.rng.randint(0, 6) if self.rng.random() < 0.4: tail += " OFFSET %d" % self.rng.randint(0, 4) return tail # -- whole queries ---------------------------------------------------- def query(self, shape): rng = self.rng t = rng.choice(self.tables) if shape == "const": self.set_scope([]) return "SELECT " + ", ".join(self.scalar_expr(False) for _ in range(rng.randint(1, 3))) if shape in ("scan", "where", "distinct"): self.set_scope([t]) sql = "SELECT " + ("DISTINCT " if shape == "distinct" else "") \ + ", ".join(self.items(False)) sql += " FROM " + t if shape in ("where", "distinct") or rng.random() < 0.4: sql += " WHERE " + self.predicate(False) if shape != "distinct": keys = [self.order_key(False) for _ in range(rng.randint(1, 2))] sql += self.order_tail(keys, ["%s.id" % t]) return sql if shape == "join": others = [x for x in self.tables if x != t] if not others: return self.query("scan") u = rng.choice(others) self.set_scope([t, u]) kind = "LEFT JOIN" if rng.random() < 0.45 else "JOIN" cands = [c for (tt, c) in self.pool_num if tt == t] or ["id"] on = rng.choice([ "%s.id = %s.id" % (t, u), "%s.id = %s.id AND %s" % (t, u, self.predicate(True)), "%s.%s = %s.id" % (t, rng.choice(cands), u), ]) grouped = rng.random() < 0.5 if grouped: gkeys = ["%s.id" % t, "%s.id" % u] sql = "SELECT " + ", ".join(self.items(True, agg=True, group_keys=gkeys)) sql += " FROM %s %s %s ON %s" % (t, kind, u, on) if rng.random() < 0.5: sql += " WHERE " + self.predicate(True) sql += " GROUP BY " + ", ".join(gkeys) sql += self.order_tail(["COUNT(*)"], gkeys) else: sql = "SELECT " + ", ".join(self.items(True, agg=False)) sql += " FROM %s %s %s ON %s" % (t, kind, u, on) if rng.random() < 0.5: sql += " WHERE " + self.predicate(True) sql += self.order_tail(["%s.id" % t], ["%s.id" % t, "%s.id" % u]) return sql if shape == "agg": self.set_scope([t]) sql = "SELECT " + ", ".join(self.items(False, agg=True)) sql += " FROM " + t if rng.random() < 0.6: sql += " WHERE " + self.predicate(False) if rng.random() < 0.4: sql += " HAVING " + self.having(None) return sql # group self.set_scope([t]) gkeys = [] for _ in range(rng.randint(1, 2)): if self.pool_all: gkeys.append(self.ref(self.pick(self.pool_all), False)) if not gkeys: gkeys = ["%s.id" % t] sql = "SELECT " + ", ".join(self.items(False, agg=True, group_keys=gkeys)) sql += " FROM " + t if rng.random() < 0.6: sql += " WHERE " + self.predicate(False) sql += " GROUP BY " + ", ".join(gkeys) if rng.random() < 0.5: sql += " HAVING " + self.having(gkeys) sql += self.order_tail(["COUNT(*)"], gkeys) return sql # -------------------------------------------------------------------------- # comparison # -------------------------------------------------------------------------- def cat(v): if v is None: return "null" if isinstance(v, bool): return "int" if isinstance(v, int): return "int" if isinstance(v, float): return "real" if isinstance(v, str): return "text" return "other:" + type(v).__name__ def same_value(a, b): return cat(a) == cat(b) and a == b def canon_row(row): out = [] for v in row: c = cat(v) out.append(("num", float(v)) if c in ("int", "real") else (c, str(v))) return tuple(out) def same_row(ra, rb): return len(ra) == len(rb) and all(same_value(a, b) for a, b in zip(ra, rb)) def compare_rows(got, want, ordered): """None when the two result sets agree, else a short description of the first difference.""" if len(got) != len(want): return "row count %d != %d" % (len(got), len(want)) if ordered: for i, (g, w) in enumerate(zip(got, want)): if not same_row(g, w): return "row %d: %r != %r" % (i, g, w) return None for g, w in zip(sorted(got, key=canon_row), sorted(want, key=canon_row)): if not same_row(g, w): return "multiset row %r != %r" % (g, w) return None # -------------------------------------------------------------------------- # runner # -------------------------------------------------------------------------- def run_battery(queries, seed, max_print=5, quiet=False): rng = random.Random(seed) con = sqlite3.connect(":memory:") shape_counts = dict.fromkeys(SHAPES, 0) mismatches, errors = [], [] rows_compared = nonempty = ordered_count = 0 for i in range(queries): shape = SHAPES[i % len(SHAPES)] spec, rows = make_schema(rng) con.close() con = sqlite3.connect(":memory:") load_sqlite(con, spec, rows) mtables = {name: minisql.table_from(cols, rows[name]) for name, cols in spec.items()} sql = Gen(rng, spec, rows).query(shape) shape_counts[shape] += 1 ordered = " ORDER BY " in sql ordered_count += 1 if ordered else 0 try: want = [list(r) for r in con.execute(sql).fetchall()] except sqlite3.Error as exc: errors.append((sql, str(exc))) continue try: _names, got = minisql.execute(sql, mtables) except Exception as exc: # noqa: BLE001 - any engine failure is red mismatches.append((shape, sql, "minisql raised %s: %s" % (type(exc).__name__, exc), None, want)) continue rows_compared += len(want) nonempty += 1 if want else 0 diff = compare_rows(got, want, ordered) if diff is not None: mismatches.append((shape, sql, diff, got, want)) con.close() report = { "queries": queries, "seed": seed, "shape_counts": shape_counts, "rows_compared": rows_compared, "nonempty": nonempty, "ordered": ordered_count, "mismatches": mismatches, "errors": errors, } if not quiet: print("minisql differential battery vs sqlite3 %s" % sqlite3.sqlite_version) print("seed=%d queries=%d" % (seed, queries)) print("shape counts: " + " ".join("%s=%d" % (k, shape_counts[k]) for k in SHAPES)) print("ordered queries: %d rows compared: %d queries returning >=1 row: %d" % (ordered_count, rows_compared, nonempty)) for shape, sql, diff, got, want in mismatches[:max_print]: print("\nMISMATCH [%s] %s\n %s" % (shape, diff, sql)) print(" oracle: %r" % (want[:6],)) print(" engine: %r" % (got[:6] if got is not None else None,)) if len(mismatches) > max_print: print("... %d more mismatches" % (len(mismatches) - max_print)) for sql, exc in errors[:max_print]: print("\nHARNESS ERROR (oracle refused generated SQL): %s\n %s" % (exc, sql)) if len(errors) > max_print: print("... %d more harness errors" % (len(errors) - max_print)) return report def check_report(report): """The non-vacuity guards. Returns a list of failure strings.""" bad = [] if report["errors"]: bad.append("%d generated statements were refused by the oracle (harness defect)" % len(report["errors"])) if report["mismatches"]: bad.append("%d mismatches" % len(report["mismatches"])) if report["queries"] < len(SHAPES): bad.append("too few queries to exercise every shape") for shape in SHAPES: if report["shape_counts"][shape] == 0: bad.append("shape %s was never exercised" % shape) if report["nonempty"] < 0.5 * report["queries"]: bad.append("only %d/%d queries returned rows -- agreement would be too close to vacuous" % (report["nonempty"], report["queries"])) if report["rows_compared"] < report["queries"]: bad.append("only %d rows compared for %d queries" % (report["rows_compared"], report["queries"])) if report["ordered"] < 0.2 * report["queries"]: bad.append("only %d ordered queries -- ORDER BY is barely covered" % report["ordered"]) return bad # -------------------------------------------------------------------------- # mutation control: prove the battery can go RED # -------------------------------------------------------------------------- def _mutants(): orig = { "compare_values": minisql.compare_values, "join_relation": minisql.join_relation, "sort_key": minisql.sort_key, "agg_sum": minisql.agg_sum, "agg_count": minisql.agg_count, "arith_div": minisql.arith_div, "_tri_and": minisql._tri_and, "like_match": minisql.like_match, } def m_null_eq(a, b, aff_a=None, aff_b=None): if a is None or b is None: return 0 # NULL = NULL is TRUE, and NULL = 1 is TRUE return orig["compare_values"](a, b, aff_a, aff_b) def m_left_inner(left, right, kind, on_eval): return orig["join_relation"](left, right, "inner", on_eval) def m_nulls_last(v): if v is None: return (3, 0) # NULLs sort last on ASC return (min(minisql.class_rank(v), 2), v) def m_sum_empty(values): r = orig["agg_sum"](values) return 0 if r is None else r def m_count_nulls(values, n_rows): return n_rows # COUNT(x) counts NULLs too def m_div_float(a, b): r = orig["arith_div"](a, b) return None if r is None else float(r) def m_and_absorb(a, b): ta, tb = minisql.tri(a), minisql.tri(b) if ta is None or tb is None: return None # NULL absorbs FALSE return 0 if (ta == 0 or tb == 0) else 1 def m_like_case_sensitive(value, pattern): if value is None or pattern is None: return None import re as _re rx = _re.compile("(?s)\\A" + "".join( ".*" if ch == "%" else ("." if ch == "_" else _re.escape(ch)) for ch in pattern) + "\\Z") return 1 if rx.match(str(value)) else 0 return [ ("null-eq", "compare_values", m_null_eq), ("left-join-as-inner", "join_relation", m_left_inner), ("nulls-last-asc", "sort_key", m_nulls_last), ("sum-empty-zero", "agg_sum", m_sum_empty), ("count-counts-nulls", "agg_count", m_count_nulls), ("int-div-becomes-real", "arith_div", m_div_float), ("and-null-absorbs-false", "_tri_and", m_and_absorb), ("like-case-sensitive", "like_match", m_like_case_sensitive), ] def run_mutants(queries, seed, quiet=False): results = [] for name, attr, fn in _mutants(): original = getattr(minisql, attr) setattr(minisql, attr, fn) try: rep = run_battery(queries, seed, max_print=0, quiet=True) finally: setattr(minisql, attr, original) caught = len(rep["mismatches"]) results.append((name, attr, caught)) if not quiet: print(" mutant %-24s patches %-15s -> %s" % (name, attr, "CAUGHT (%d mismatches)" % caught if caught else "SURVIVED -- battery has no teeth here")) return results # -------------------------------------------------------------------------- def main(argv=None): ap = argparse.ArgumentParser(description="minisql vs sqlite3 differential battery") ap.add_argument("--queries", type=int, default=1400) ap.add_argument("--seed", type=int, default=20261022) ap.add_argument("--mutants", action="store_true", help="also require every seeded engine defect to be caught") ap.add_argument("--mutant-queries", type=int, default=140) ap.add_argument("--max-print", type=int, default=5) ap.add_argument("--quiet", action="store_true") args = ap.parse_args(argv) rep = run_battery(args.queries, args.seed, max_print=args.max_print, quiet=args.quiet) bad = check_report(rep) mutants_caught = 0 if args.mutants: if not args.quiet: print("\nmutation control (every seeded defect must be caught):") results = run_mutants(args.mutant_queries, args.seed, quiet=args.quiet) survived = [n for n, _a, c in results if not c] mutants_caught = len(results) - len(survived) if not args.quiet: print("mutants caught: %d/%d" % (mutants_caught, len(results))) if survived: bad.append("mutants survived (battery has no teeth): " + ", ".join(survived)) if not args.quiet: print() for b in bad: print("FAIL: " + b) if not bad: print("all %d queries agreed with sqlite3; %d rows compared; %d mutants caught" % (rep["queries"], rep["rows_compared"], mutants_caught)) print("RESULT: " + ("FAIL" if bad else "PASS")) return 1 if bad else 0 if __name__ == "__main__": sys.exit(main())