752 lines
33 KiB
Python
752 lines
33 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
diff_sqlite.py -- randomized differential battery: minisql.py vs sqlite3 (the oracle).
|
|
|
|
Wizard: null (lane: Codeforge -- hard coding and agentic reasoning).
|
|
|
|
What it does, per query:
|
|
1. generates a random schema (2-3 tables, INTEGER/TEXT/REAL columns, NULLs, 0-12 rows),
|
|
2. generates one random SELECT in the supported grammar,
|
|
3. creates the SAME schema in an in-memory sqlite3 database and executes the SAME SQL text
|
|
in both engines,
|
|
4. compares the result sets -- strictly, by type category (int / real / text / null) and by
|
|
value -- as an ordered sequence when the query has ORDER BY, else as a multiset.
|
|
|
|
It exits 0 only when ALL of these hold, so the exit code cannot pass vacuously:
|
|
* sqlite3 is importable and every generated statement ran on BOTH engines without error,
|
|
* the requested number of queries actually executed,
|
|
* every query shape was exercised (const / scan / where / join / group / agg / distinct),
|
|
* at least a set fraction of queries returned at least one row, and the battery compared at
|
|
least a set number of rows (so agreement is not agreement about empty sets),
|
|
* zero mismatches,
|
|
* with --mutants: every seeded engine defect is CAUGHT (each mutant must produce at least
|
|
one mismatch). A mutant that survives means this battery has no teeth for that semantics.
|
|
|
|
Generator constraints (each one is there to keep the oracle deterministic, and each is a
|
|
documented hole rather than an accident):
|
|
* every ORDER BY is made total by appending the `id` columns of the FROM tables, so ties
|
|
cannot reorder differently in the two engines;
|
|
* DISTINCT queries are never ordered (SQLite requires ORDER BY terms to be in the result
|
|
set, and ordered ties would not be comparable) -- they are compared as multisets;
|
|
* LIMIT/OFFSET only appear with an ORDER BY;
|
|
* with GROUP BY, the select list is group keys plus aggregates only -- never a bare column
|
|
of a multi-row group, whose value SQLite leaves unspecified;
|
|
* the NULL-tri-value families (`FALSE AND NULL`, `TRUE AND NULL`, `ON ... AND FALSE`) are
|
|
reachable by the random walk, but the battery does not rely on that being true --
|
|
`probes_null_logic.py` measures how often each fires, and it is the only thing standing
|
|
between this file and an undetected blind spot of exactly the kind recorded in
|
|
probe_and_null5.py.
|
|
|
|
Usage:
|
|
python3 diff_sqlite.py [--queries N] [--seed S] [--mutants] [--max-print K] [--quiet]
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import random
|
|
import sqlite3
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import minisql # noqa: E402
|
|
|
|
SHAPES = ("const", "scan", "where", "join", "group", "agg", "distinct")
|
|
|
|
# Every shape gets a FIXED slice of the run, not `i % len(SHAPES)` which needs a whole
|
|
# multiple of 7 to be even. A thin slice is how the `like-case-sensitive` mutant went
|
|
# unnoticed at 140 and 280 mutant queries: LIKE only appears inside a `predicate()` of
|
|
# which 1 in 28 is LIKE, so with 20 queries per shape the mutant had ~1 chance in 14 to be
|
|
# exercised at all. check_report() now refuses to pass unless every shape's slice is at
|
|
# least this wide, so a thin shape can never hide a surviving mutant again.
|
|
_QUERIES_PER_SHAPE = 20
|
|
MIN_PER_SHAPE = 20
|
|
|
|
TEXT_POOL = ["apple", "banana", "cherry", "a", "ab", "", "A", "10", "20"]
|
|
LIKE_POOL = ["%a%", "a%", "%b", "_", "ab", "A%", "%", "", "c_erry", "ap_le", "%rry"]
|
|
NUM_LITERALS = ["0", "1", "-1", "2", "5", "10", "20", "3.5", "5.5", "0.0"]
|
|
STR_NUM_LITERALS = ["0", "5", "10", "20", "5.5", "abc", ""]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# schema
|
|
# --------------------------------------------------------------------------
|
|
|
|
def make_schema(rng):
|
|
"""Return (spec, rows): spec is {table: [(col, declared_type)]}, rows is {table: [[v...]]}.
|
|
|
|
`id` is a unique non-NULL INTEGER in every table; it is the tiebreaker the generator
|
|
appends to every ORDER BY, so ordered results are deterministic in both engines.
|
|
"""
|
|
spec = {
|
|
"t1": [("id", "INTEGER"), ("a", "INTEGER"), ("b", "TEXT"), ("c", "REAL")],
|
|
"t2": [("id", "INTEGER"), ("d", "INTEGER"), ("e", "TEXT")],
|
|
}
|
|
if rng.random() < 0.5:
|
|
spec["t3"] = [("id", "INTEGER"), ("f", "REAL"), ("g", "TEXT")]
|
|
|
|
rows = {}
|
|
for name, cols in spec.items():
|
|
n = rng.randint(0, 12)
|
|
table_rows = []
|
|
for i in range(1, n + 1):
|
|
row = []
|
|
for cname, ctype in cols:
|
|
if cname == "id":
|
|
row.append(i)
|
|
elif rng.random() < 0.2:
|
|
row.append(None)
|
|
elif ctype == "INTEGER":
|
|
row.append(rng.randint(0, 20))
|
|
elif ctype == "REAL":
|
|
row.append(round(rng.uniform(0, 10), 1))
|
|
else:
|
|
row.append(rng.choice(TEXT_POOL))
|
|
table_rows.append(row)
|
|
rows[name] = table_rows
|
|
return spec, rows
|
|
|
|
|
|
def load_sqlite(con, spec, rows):
|
|
for name, cols in spec.items():
|
|
con.execute("CREATE TABLE %s (%s)" % (
|
|
name, ", ".join("%s %s" % (c, t) for c, t in cols)))
|
|
if rows[name]:
|
|
con.executemany("INSERT INTO %s VALUES (%s)" % (
|
|
name, ", ".join("?" * len(cols))), rows[name])
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# query generator
|
|
# --------------------------------------------------------------------------
|
|
|
|
class Gen:
|
|
"""Emits one random SELECT. All column references are drawn from the tables named in
|
|
`scope`, which the query method sets to exactly the FROM tables before generating."""
|
|
|
|
def __init__(self, rng, spec, rows):
|
|
self.rng = rng
|
|
self.spec = spec
|
|
self.rows = rows
|
|
self.tables = list(spec)
|
|
self.scope = []
|
|
self.pool_num = [] # [(table, col)] with INTEGER/REAL type, inside scope
|
|
self.pool_text = [] # [(table, col)] with TEXT type, inside scope
|
|
self.pool_all = [] # [(table, col)] any type except id
|
|
self.pool_id = [] # [(table, "id")]
|
|
|
|
# -- scope ------------------------------------------------------------
|
|
def set_scope(self, tables):
|
|
self.scope = list(tables)
|
|
self.pool_num, self.pool_text, self.pool_all, self.pool_id = [], [], [], []
|
|
for t in self.scope:
|
|
for c, ty in self.spec[t]:
|
|
if c == "id":
|
|
self.pool_id.append((t, c))
|
|
else:
|
|
self.pool_all.append((t, c))
|
|
(self.pool_num if ty in ("INTEGER", "REAL") else self.pool_text).append((t, c))
|
|
|
|
def ref(self, tc, qualify):
|
|
t, c = tc
|
|
return "%s.%s" % (t, c) if qualify else c
|
|
|
|
def pick(self, pool):
|
|
return self.rng.choice(pool) if pool else None
|
|
|
|
# -- literals ---------------------------------------------------------
|
|
def num_literal(self):
|
|
return self.rng.choice(NUM_LITERALS)
|
|
|
|
def text_literal(self):
|
|
return "'%s'" % self.rng.choice(TEXT_POOL)
|
|
|
|
# -- expressions ------------------------------------------------------
|
|
def numeric_expr(self, qualify, depth=0):
|
|
pool = self.pool_num
|
|
r = self.rng.random()
|
|
if not pool or r < 0.4 or depth >= 2:
|
|
return self.ref(self.pick(pool), qualify) if pool else self.num_literal()
|
|
if r < 0.55:
|
|
return self.num_literal()
|
|
if r < 0.68:
|
|
return "(%s + %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal())
|
|
if r < 0.8:
|
|
return "(%s * %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal())
|
|
if r < 0.9:
|
|
return "(%s - %s)" % (self.numeric_expr(qualify, depth + 1), self.num_literal())
|
|
# division is here on purpose: SQLite's `/` stays INTEGER for two integers, and the
|
|
# `int-div-becomes-real` mutant exists to prove this battery notices when it does not.
|
|
return "(%s / %s)" % (self.numeric_expr(qualify, depth + 1),
|
|
self.rng.choice(["2", "3", "1", "4"]))
|
|
|
|
def scalar_expr(self, qualify, depth=0):
|
|
r = self.rng.random()
|
|
if r < 0.45:
|
|
return self.numeric_expr(qualify, depth)
|
|
if r < 0.6 and self.pool_text:
|
|
return self.ref(self.pick(self.pool_text), qualify)
|
|
if r < 0.7:
|
|
return self.text_literal()
|
|
if r < 0.8 and depth < 2:
|
|
return "CASE WHEN %s THEN %s ELSE %s END" % (
|
|
self.predicate(qualify, depth + 1), self.scalar_expr(qualify, depth + 1),
|
|
self.scalar_expr(qualify, depth + 1))
|
|
return self.predicate(qualify, depth + 1)
|
|
|
|
def predicate(self, qualify, depth=0):
|
|
r = self.rng.random()
|
|
if depth >= 2:
|
|
r = min(r, 0.66)
|
|
if r < 0.35:
|
|
return self.comparison(qualify)
|
|
if r < 0.45:
|
|
return self.conjunction(qualify)
|
|
if r < 0.55:
|
|
return "(%s OR %s)" % (self.predicate(qualify, depth + 1),
|
|
self.predicate(qualify, depth + 1))
|
|
if r < 0.6:
|
|
return "NOT %s" % self.predicate(qualify, depth + 1)
|
|
if r < 0.68 and self.pool_all:
|
|
tc = self.pick(self.pool_all)
|
|
return "%s IS%s NULL" % (self.ref(tc, qualify),
|
|
"" if self.rng.random() < 0.5 else " NOT")
|
|
if r < 0.82 and self.pool_num:
|
|
items = ", ".join(self.num_literal() for _ in range(self.rng.randint(1, 3)))
|
|
return "%s%s IN (%s)" % (self.ref(self.pick(self.pool_num), qualify),
|
|
" NOT" if self.rng.random() < 0.3 else "", items)
|
|
if r < 0.88 and self.pool_text:
|
|
items = ", ".join(self.text_literal() for _ in range(self.rng.randint(1, 3)))
|
|
return "%s%s IN (%s)" % (self.ref(self.pick(self.pool_text), qualify),
|
|
" NOT" if self.rng.random() < 0.3 else "", items)
|
|
if r < 0.93 and self.pool_num:
|
|
return "%s%s BETWEEN %s AND %s" % (
|
|
self.ref(self.pick(self.pool_num), qualify),
|
|
" NOT" if self.rng.random() < 0.3 else "",
|
|
self.num_literal(), self.num_literal())
|
|
if self.pool_text:
|
|
return "%s%s LIKE '%s'" % (self.ref(self.pick(self.pool_text), qualify),
|
|
" NOT" if self.rng.random() < 0.3 else "",
|
|
self.rng.choice(LIKE_POOL))
|
|
return self.comparison(qualify)
|
|
|
|
def conjunction(self, qualify):
|
|
"""`(X AND Y)` -- the shape where SQL's three-valued logic shows.
|
|
|
|
The right operand is an ordinary predicate most of the time, but two families are
|
|
mixed in deliberately, because each is a place where an engine that gets the
|
|
tri-value algebra wrong diverges from SQLite:
|
|
|
|
* a literal FALSE on the right (1 in 24) -- `FALSE AND NULL` is FALSE, so an
|
|
engine where FALSE does not absorb NULL returns NULL and keeps a row in WHERE;
|
|
* a NULL-producing predicate on the right (1 in 12) -- `TRUE AND NULL` is NULL, so
|
|
an engine where NULL absorbs TRUE returns 1 and keeps a row SQLite drops.
|
|
|
|
Both are reachable by the random walk. probes_null_logic.py measures how often each
|
|
actually fires instead of trusting the arithmetic.
|
|
"""
|
|
r = self.rng.random()
|
|
if r < 1 / 24:
|
|
right = "1 = 0"
|
|
elif r < 1 / 24 + 1 / 12 and self.pool_all:
|
|
right = self.null_predicate(qualify)
|
|
else:
|
|
right = self.predicate(qualify, 1)
|
|
return "(%s AND %s)" % (self.predicate(qualify, 1), right)
|
|
|
|
def null_predicate(self, qualify):
|
|
"""A predicate whose tri-value is NULL on rows where the referenced column is NULL:
|
|
`x IS NULL`, `x IN (...)`, `x BETWEEN ... AND ...`, `x LIKE ...`."""
|
|
tc = self.pick(self.pool_all)
|
|
if tc is None:
|
|
return self.comparison(qualify)
|
|
r = self.rng.random()
|
|
if r < 0.4:
|
|
return "%s IS NULL" % self.ref(tc, qualify)
|
|
if r < 0.6:
|
|
items = ", ".join(self.num_literal() for _ in range(self.rng.randint(1, 2)))
|
|
return "%s IN (%s)" % (self.ref(tc, qualify), items)
|
|
if r < 0.8:
|
|
return "%s BETWEEN %s AND %s" % (self.ref(tc, qualify),
|
|
self.num_literal(), self.num_literal())
|
|
return "%s LIKE '%s'" % (self.ref(tc, qualify), self.rng.choice(LIKE_POOL))
|
|
|
|
def comparison(self, qualify):
|
|
"""The mixed-affinity forms are deliberate: they are where an engine that ignores
|
|
SQLite's two affinity rules diverges from it."""
|
|
r = self.rng.random()
|
|
op = self.rng.choice(["=", "<>", "<", "<=", ">", ">="])
|
|
if r < 0.35 and self.pool_num:
|
|
return "%s %s %s" % (self.ref(self.pick(self.pool_num), qualify), op,
|
|
self.num_literal())
|
|
if r < 0.55 and self.pool_text:
|
|
return "%s %s %s" % (self.ref(self.pick(self.pool_text), qualify), op,
|
|
self.text_literal())
|
|
if r < 0.7 and self.pool_num:
|
|
# numeric affinity is applied to the text literal
|
|
return "%s %s '%s'" % (self.ref(self.pick(self.pool_num), qualify), op,
|
|
self.rng.choice(STR_NUM_LITERALS))
|
|
if r < 0.85 and self.pool_text:
|
|
# TEXT affinity is applied to the numeric literal
|
|
return "%s %s %s" % (self.ref(self.pick(self.pool_text), qualify), op,
|
|
self.rng.choice(["0", "5", "10", "20", "-1"]))
|
|
if self.pool_num:
|
|
return "%s %s %s" % (self.numeric_expr(qualify, 1), op, self.num_literal())
|
|
return self.comparison(qualify) if (self.pool_text or self.pool_num) else "1 = 1"
|
|
|
|
# -- select items -----------------------------------------------------
|
|
def agg_item(self, qualify, i):
|
|
fn = self.rng.choice(["COUNT", "SUM", "AVG", "MIN", "MAX"])
|
|
if fn == "COUNT" and self.rng.random() < 0.4:
|
|
return "COUNT(*) AS agg%d" % i
|
|
pool = self.pool_num if (fn in ("SUM", "AVG") and self.pool_num) else self.pool_all
|
|
if not pool:
|
|
return "COUNT(*) AS agg%d" % i
|
|
return "%s(%s) AS agg%d" % (fn, self.ref(self.pick(pool), qualify), i)
|
|
|
|
def items(self, qualify, agg=False, group_keys=None):
|
|
out = []
|
|
if not agg and self.rng.random() < 0.25:
|
|
out.append("*")
|
|
for i in range(self.rng.randint(1, 3)):
|
|
if agg:
|
|
if group_keys and self.rng.random() < 0.35:
|
|
out.append("%s AS gk%d" % (self.rng.choice(group_keys), i))
|
|
else:
|
|
out.append(self.agg_item(qualify, i))
|
|
else:
|
|
e = self.scalar_expr(qualify)
|
|
out.append("%s AS x%d" % (e, i) if self.rng.random() < 0.6 else e)
|
|
if not out:
|
|
out.append("*")
|
|
return out
|
|
|
|
def having(self, gkeys):
|
|
r = self.rng.random()
|
|
if r < 0.45:
|
|
return "COUNT(*) > %d" % self.rng.randint(0, 3)
|
|
if r < 0.6 and self.pool_num:
|
|
return "SUM(%s) > %s" % (self.ref(self.pick(self.pool_num), False),
|
|
self.rng.choice(["5", "10", "20", "0"]))
|
|
if r < 0.7 and self.pool_num:
|
|
return "AVG(%s) > %s" % (self.ref(self.pick(self.pool_num), False),
|
|
self.rng.choice(["1", "5", "10"]))
|
|
if r < 0.8 and self.pool_all:
|
|
return "MAX(%s) > %s" % (self.ref(self.pick(self.pool_all), False),
|
|
self.rng.choice(["5", "10", "1.5"]))
|
|
if r < 0.9 and gkeys:
|
|
return "%s IS NOT NULL" % self.rng.choice(gkeys)
|
|
return "COUNT(*) > 1"
|
|
|
|
def order_key(self, qualify):
|
|
"""ORDER BY terms are always column references: a bare integer literal in ORDER BY is
|
|
an ORDINAL in SQLite, not a constant, and the oracle rejects it out of range."""
|
|
pool = list(self.pool_all) + list(self.pool_id)
|
|
if not pool:
|
|
return "1 = 1"
|
|
return self.ref(self.rng.choice(pool), qualify)
|
|
|
|
def order_tail(self, keys, tiebreak):
|
|
"""keys: ordering exprs; tiebreak: exprs that make the order total."""
|
|
terms = ["%s%s" % (k, " DESC" if self.rng.random() < 0.4 else "") for k in keys]
|
|
terms.extend(tiebreak)
|
|
tail = " ORDER BY " + ", ".join(terms)
|
|
if self.rng.random() < 0.35:
|
|
tail += " LIMIT %d" % self.rng.randint(0, 6)
|
|
if self.rng.random() < 0.4:
|
|
tail += " OFFSET %d" % self.rng.randint(0, 4)
|
|
return tail
|
|
|
|
# -- whole queries ----------------------------------------------------
|
|
def query(self, shape):
|
|
rng = self.rng
|
|
t = rng.choice(self.tables)
|
|
|
|
if shape == "const":
|
|
self.set_scope([])
|
|
return "SELECT " + ", ".join(self.scalar_expr(False) for _ in range(rng.randint(1, 3)))
|
|
|
|
if shape in ("scan", "where", "distinct"):
|
|
self.set_scope([t])
|
|
sql = "SELECT " + ("DISTINCT " if shape == "distinct" else "") \
|
|
+ ", ".join(self.items(False))
|
|
sql += " FROM " + t
|
|
if shape in ("where", "distinct") or rng.random() < 0.4:
|
|
sql += " WHERE " + self.predicate(False)
|
|
if shape != "distinct":
|
|
keys = [self.order_key(False) for _ in range(rng.randint(1, 2))]
|
|
sql += self.order_tail(keys, ["%s.id" % t])
|
|
return sql
|
|
|
|
if shape == "join":
|
|
others = [x for x in self.tables if x != t]
|
|
if not others:
|
|
return self.query("scan")
|
|
u = rng.choice(others)
|
|
self.set_scope([t, u])
|
|
kind = "LEFT JOIN" if rng.random() < 0.45 else "JOIN"
|
|
cands = [c for (tt, c) in self.pool_num if tt == t] or ["id"]
|
|
on = rng.choice([
|
|
"%s.id = %s.id" % (t, u),
|
|
"%s.id = %s.id AND %s" % (t, u, self.predicate(True)),
|
|
"%s.%s = %s.id" % (t, rng.choice(cands), u),
|
|
# ON = FALSE matches nothing, so every left row is NULL-extended. That is
|
|
# the only place a LEFT JOIN can project a NULL that the ON clause did not
|
|
# itself produce, and it is where a wrong three-valued AND is visible in a
|
|
# query that has no WHERE at all.
|
|
"%s.id = %s.id AND 1 = 0" % (t, u),
|
|
])
|
|
grouped = rng.random() < 0.5
|
|
if grouped:
|
|
gkeys = ["%s.id" % t, "%s.id" % u]
|
|
sql = "SELECT " + ", ".join(self.items(True, agg=True, group_keys=gkeys))
|
|
sql += " FROM %s %s %s ON %s" % (t, kind, u, on)
|
|
if rng.random() < 0.5:
|
|
sql += " WHERE " + self.predicate(True)
|
|
sql += " GROUP BY " + ", ".join(gkeys)
|
|
sql += self.order_tail(["COUNT(*)"], gkeys)
|
|
else:
|
|
sql = "SELECT " + ", ".join(self.items(True, agg=False))
|
|
sql += " FROM %s %s %s ON %s" % (t, kind, u, on)
|
|
if rng.random() < 0.5:
|
|
sql += " WHERE " + self.predicate(True)
|
|
sql += self.order_tail(["%s.id" % t], ["%s.id" % t, "%s.id" % u])
|
|
return sql
|
|
|
|
if shape == "agg":
|
|
self.set_scope([t])
|
|
sql = "SELECT " + ", ".join(self.items(False, agg=True))
|
|
sql += " FROM " + t
|
|
if rng.random() < 0.6:
|
|
sql += " WHERE " + self.predicate(False)
|
|
if rng.random() < 0.4:
|
|
sql += " HAVING " + self.having(None)
|
|
return sql
|
|
|
|
# group
|
|
self.set_scope([t])
|
|
gkeys = []
|
|
for _ in range(rng.randint(1, 2)):
|
|
if self.pool_all:
|
|
gkeys.append(self.ref(self.pick(self.pool_all), False))
|
|
if not gkeys:
|
|
gkeys = ["%s.id" % t]
|
|
sql = "SELECT " + ", ".join(self.items(False, agg=True, group_keys=gkeys))
|
|
sql += " FROM " + t
|
|
if rng.random() < 0.6:
|
|
sql += " WHERE " + self.predicate(False)
|
|
sql += " GROUP BY " + ", ".join(gkeys)
|
|
if rng.random() < 0.5:
|
|
sql += " HAVING " + self.having(gkeys)
|
|
sql += self.order_tail(["COUNT(*)"], gkeys)
|
|
return sql
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# comparison
|
|
# --------------------------------------------------------------------------
|
|
|
|
def cat(v):
|
|
if v is None:
|
|
return "null"
|
|
if isinstance(v, bool):
|
|
return "int"
|
|
if isinstance(v, int):
|
|
return "int"
|
|
if isinstance(v, float):
|
|
return "real"
|
|
if isinstance(v, str):
|
|
return "text"
|
|
return "other:" + type(v).__name__
|
|
|
|
|
|
def same_value(a, b):
|
|
return cat(a) == cat(b) and a == b
|
|
|
|
|
|
def canon_row(row):
|
|
out = []
|
|
for v in row:
|
|
c = cat(v)
|
|
out.append(("num", float(v)) if c in ("int", "real") else (c, str(v)))
|
|
return tuple(out)
|
|
|
|
|
|
def same_row(ra, rb):
|
|
return len(ra) == len(rb) and all(same_value(a, b) for a, b in zip(ra, rb))
|
|
|
|
|
|
def compare_rows(got, want, ordered):
|
|
"""None when the two result sets agree, else a short description of the first difference."""
|
|
if len(got) != len(want):
|
|
return "row count %d != %d" % (len(got), len(want))
|
|
if ordered:
|
|
for i, (g, w) in enumerate(zip(got, want)):
|
|
if not same_row(g, w):
|
|
return "row %d: %r != %r" % (i, g, w)
|
|
return None
|
|
for g, w in zip(sorted(got, key=canon_row), sorted(want, key=canon_row)):
|
|
if not same_row(g, w):
|
|
return "multiset row %r != %r" % (g, w)
|
|
return None
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# runner
|
|
# --------------------------------------------------------------------------
|
|
|
|
def run_battery(queries, seed, max_print=5, quiet=False):
|
|
rng = random.Random(seed)
|
|
con = sqlite3.connect(":memory:")
|
|
shape_counts = dict.fromkeys(SHAPES, 0)
|
|
mismatches, errors = [], []
|
|
rows_compared = nonempty = ordered_count = 0
|
|
|
|
for i in range(queries):
|
|
shape = SHAPES[(i // _QUERIES_PER_SHAPE) % len(SHAPES)]
|
|
spec, rows = make_schema(rng)
|
|
con.close()
|
|
con = sqlite3.connect(":memory:")
|
|
load_sqlite(con, spec, rows)
|
|
mtables = {name: minisql.table_from(cols, rows[name]) for name, cols in spec.items()}
|
|
sql = Gen(rng, spec, rows).query(shape)
|
|
shape_counts[shape] += 1
|
|
ordered = " ORDER BY " in sql
|
|
ordered_count += 1 if ordered else 0
|
|
try:
|
|
want = [list(r) for r in con.execute(sql).fetchall()]
|
|
except sqlite3.Error as exc:
|
|
errors.append((sql, str(exc)))
|
|
continue
|
|
try:
|
|
_names, got = minisql.execute(sql, mtables)
|
|
except Exception as exc: # noqa: BLE001 - any engine failure is red
|
|
mismatches.append((shape, sql, "minisql raised %s: %s" % (type(exc).__name__, exc),
|
|
None, want))
|
|
continue
|
|
rows_compared += len(want)
|
|
nonempty += 1 if want else 0
|
|
diff = compare_rows(got, want, ordered)
|
|
if diff is not None:
|
|
mismatches.append((shape, sql, diff, got, want))
|
|
con.close()
|
|
|
|
report = {
|
|
"queries": queries, "seed": seed, "shape_counts": shape_counts,
|
|
"rows_compared": rows_compared, "nonempty": nonempty, "ordered": ordered_count,
|
|
"mismatches": mismatches, "errors": errors, "mutant_queries": None,
|
|
# run_battery() does not run mutants; it only measures engine-vs-oracle agreement.
|
|
# Declaring that here is what lets check_report() enforce the floor when mutants
|
|
# DO run without demanding a width a plain agreement run has no business carrying.
|
|
"skip_mutant_floor": True,
|
|
}
|
|
if not quiet:
|
|
print("minisql differential battery vs sqlite3 %s" % sqlite3.sqlite_version)
|
|
print("seed=%d queries=%d" % (seed, queries))
|
|
print("shape counts: " + " ".join("%s=%d" % (k, shape_counts[k]) for k in SHAPES))
|
|
print("ordered queries: %d rows compared: %d queries returning >=1 row: %d"
|
|
% (ordered_count, rows_compared, nonempty))
|
|
for shape, sql, diff, got, want in mismatches[:max_print]:
|
|
print("\nMISMATCH [%s] %s\n %s" % (shape, diff, sql))
|
|
print(" oracle: %r" % (want[:6],))
|
|
print(" engine: %r" % (got[:6] if got is not None else None,))
|
|
if len(mismatches) > max_print:
|
|
print("... %d more mismatches" % (len(mismatches) - max_print))
|
|
for sql, exc in errors[:max_print]:
|
|
print("\nHARNESS ERROR (oracle refused generated SQL): %s\n %s" % (exc, sql))
|
|
if len(errors) > max_print:
|
|
print("... %d more harness errors" % (len(errors) - max_print))
|
|
return report
|
|
|
|
|
|
def check_report(report):
|
|
"""The non-vacuity guards. Returns a list of failure strings."""
|
|
bad = []
|
|
if report["errors"]:
|
|
bad.append("%d generated statements were refused by the oracle (harness defect)"
|
|
% len(report["errors"]))
|
|
if report["mismatches"]:
|
|
bad.append("%d mismatches" % len(report["mismatches"]))
|
|
if report["queries"] < len(SHAPES):
|
|
bad.append("too few queries to exercise every shape")
|
|
for shape in SHAPES:
|
|
if report["shape_counts"][shape] == 0:
|
|
bad.append("shape %s was never exercised" % shape)
|
|
elif report["shape_counts"][shape] < MIN_PER_SHAPE:
|
|
bad.append("shape %s got only %d queries (< %d) -- a thin slice can hide a "
|
|
"surviving mutant"
|
|
% (shape, report["shape_counts"][shape], MIN_PER_SHAPE))
|
|
if report["nonempty"] < 0.5 * report["queries"]:
|
|
bad.append("only %d/%d queries returned rows -- agreement would be too close to vacuous"
|
|
% (report["nonempty"], report["queries"]))
|
|
if report["rows_compared"] < report["queries"]:
|
|
bad.append("only %d rows compared for %d queries"
|
|
% (report["rows_compared"], report["queries"]))
|
|
if report["ordered"] < 0.2 * report["queries"]:
|
|
bad.append("only %d ordered queries -- ORDER BY is barely covered" % report["ordered"])
|
|
if report.get("skip_mutant_floor") is not True:
|
|
if report.get("mutant_queries") is None:
|
|
bad.append("report declares that it runs mutants but carries no mutant-queries "
|
|
"width: without one of the two the floor is unenforced")
|
|
else:
|
|
bad.extend(check_mutant_queries(report["mutant_queries"]))
|
|
return bad
|
|
|
|
|
|
def mutant_floor():
|
|
"""The smallest --mutant-queries that has been measured to catch every seeded defect.
|
|
|
|
This is not a constant someone guessed. Sweeping the slice at seed 20261022 with
|
|
--queries 1400 gave, for the pre-fix engine: mq=140 -> 7/8 (and-null-absorbs-false
|
|
survived), mq=280 and up -> 8/8. The post-fix engine catches 8/8 at every width
|
|
measured, 140 through 1400. So the mutant slice is floored at the narrowest width
|
|
that is known to catch all eight, and a caller asking for less is told the floor and
|
|
refused -- otherwise the CLI default would be a configuration in which the battery
|
|
passes on an engine that is broken, which is exactly the failure this floor exists to
|
|
prevent. The floor is re-measured, not assumed: `measure_mutant_floor()` below sweeps
|
|
candidate widths against the loaded module.
|
|
"""
|
|
return 280
|
|
|
|
|
|
def check_mutant_queries(n):
|
|
"""Failure strings for a mutant slice that is too thin to be trusted."""
|
|
floor = mutant_floor()
|
|
if n < floor:
|
|
return ["--mutant-queries %d is below the measured floor %d: at the narrowest width "
|
|
"measured to catch all seeded defects the floor is %d, and a thinner slice "
|
|
"lets a surviving mutant read as PASS" % (n, floor, floor)]
|
|
return []
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# mutation control: prove the battery can go RED
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _mutants():
|
|
orig = {
|
|
"compare_values": minisql.compare_values,
|
|
"join_relation": minisql.join_relation,
|
|
"sort_key": minisql.sort_key,
|
|
"agg_sum": minisql.agg_sum,
|
|
"agg_count": minisql.agg_count,
|
|
"arith_div": minisql.arith_div,
|
|
"_tri_and": minisql._tri_and,
|
|
"like_match": minisql.like_match,
|
|
}
|
|
|
|
def m_null_eq(a, b, aff_a=None, aff_b=None):
|
|
if a is None or b is None:
|
|
return 0 # NULL = NULL is TRUE, and NULL = 1 is TRUE
|
|
return orig["compare_values"](a, b, aff_a, aff_b)
|
|
|
|
def m_left_inner(left, right, kind, on_eval):
|
|
return orig["join_relation"](left, right, "inner", on_eval)
|
|
|
|
def m_nulls_last(v):
|
|
if v is None:
|
|
return (3, 0) # NULLs sort last on ASC
|
|
return (min(minisql.class_rank(v), 2), v)
|
|
|
|
def m_sum_empty(values):
|
|
r = orig["agg_sum"](values)
|
|
return 0 if r is None else r
|
|
|
|
def m_count_nulls(values, n_rows):
|
|
return n_rows # COUNT(x) counts NULLs too
|
|
|
|
def m_div_float(a, b):
|
|
r = orig["arith_div"](a, b)
|
|
return None if r is None else float(r)
|
|
|
|
def m_and_absorb(a, b):
|
|
ta, tb = minisql.tri(a), minisql.tri(b)
|
|
if ta is None or tb is None:
|
|
return None # NULL absorbs FALSE
|
|
return 0 if (ta == 0 or tb == 0) else 1
|
|
|
|
def m_like_case_sensitive(value, pattern):
|
|
if value is None or pattern is None:
|
|
return None
|
|
import re as _re
|
|
rx = _re.compile("(?s)\\A" + "".join(
|
|
".*" if ch == "%" else ("." if ch == "_" else _re.escape(ch))
|
|
for ch in pattern) + "\\Z")
|
|
return 1 if rx.match(str(value)) else 0
|
|
|
|
return [
|
|
("null-eq", "compare_values", m_null_eq),
|
|
("left-join-as-inner", "join_relation", m_left_inner),
|
|
("nulls-last-asc", "sort_key", m_nulls_last),
|
|
("sum-empty-zero", "agg_sum", m_sum_empty),
|
|
("count-counts-nulls", "agg_count", m_count_nulls),
|
|
("int-div-becomes-real", "arith_div", m_div_float),
|
|
("and-null-absorbs-false", "_tri_and", m_and_absorb),
|
|
("like-case-sensitive", "like_match", m_like_case_sensitive),
|
|
]
|
|
|
|
|
|
def run_mutants(queries, seed, quiet=False):
|
|
results = []
|
|
for name, attr, fn in _mutants():
|
|
original = getattr(minisql, attr)
|
|
setattr(minisql, attr, fn)
|
|
try:
|
|
rep = run_battery(queries, seed, max_print=0, quiet=True)
|
|
finally:
|
|
setattr(minisql, attr, original)
|
|
caught = len(rep["mismatches"])
|
|
results.append((name, attr, caught))
|
|
if not quiet:
|
|
print(" mutant %-24s patches %-15s -> %s"
|
|
% (name, attr, "CAUGHT (%d mismatches)" % caught if caught
|
|
else "SURVIVED -- battery has no teeth here"))
|
|
return results
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description="minisql vs sqlite3 differential battery")
|
|
ap.add_argument("--queries", type=int, default=1400)
|
|
ap.add_argument("--seed", type=int, default=20261022)
|
|
ap.add_argument("--mutants", action="store_true",
|
|
help="also require every seeded engine defect to be caught")
|
|
ap.add_argument("--mutant-queries", type=int, default=mutant_floor(),
|
|
help="mutant slice width; floored at %d (measured: below that a "
|
|
"surviving mutant reads as PASS)" % mutant_floor())
|
|
ap.add_argument("--max-print", type=int, default=5)
|
|
ap.add_argument("--quiet", action="store_true")
|
|
args = ap.parse_args(argv)
|
|
|
|
rep = run_battery(args.queries, args.seed, max_print=args.max_print, quiet=args.quiet)
|
|
bad = check_report(rep)
|
|
|
|
mutants_caught = 0
|
|
if args.mutants:
|
|
bad.extend(check_mutant_queries(args.mutant_queries))
|
|
if not args.quiet:
|
|
print("\nmutation control (every seeded engine defect must be caught):")
|
|
results = run_mutants(args.mutant_queries, args.seed, quiet=args.quiet)
|
|
survived = [n for n, _a, c in results if not c]
|
|
mutants_caught = len(results) - len(survived)
|
|
if not args.quiet:
|
|
print("mutants caught: %d/%d" % (mutants_caught, len(results)))
|
|
if survived:
|
|
bad.append("mutants survived (battery has no teeth): " + ", ".join(survived))
|
|
|
|
if not args.quiet:
|
|
print()
|
|
for b in bad:
|
|
print("FAIL: " + b)
|
|
if not bad:
|
|
print("all %d queries agreed with sqlite3; %d rows compared; %d mutants caught"
|
|
% (rep["queries"], rep["rows_compared"], mutants_caught))
|
|
print("RESULT: " + ("FAIL" if bad else "PASS"))
|
|
return 1 if bad else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|