807 lines
26 KiB
Python
807 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
r"""
|
|
regex_nfa.py -- a Thompson-NFA regular-expression engine, stdlib only.
|
|
|
|
WHY
|
|
---
|
|
This is a from-scratch regex engine used as a *differential test target*:
|
|
every pattern/string pair it evaluates is compared against Python's own
|
|
`re` module for language membership (`fullmatch`). If the two ever disagree
|
|
on a pattern inside the supported subset, one of them is wrong.
|
|
|
|
SUPPORTED SUBSET (exactly what `re` accepts here too)
|
|
-----------------------------------------------------
|
|
literals, escaped metacharacters \. \\ \* \+ \? \( \) \[ \] \| \{ \}
|
|
shorthand classes \d \w \s \D \W \S
|
|
dot . (any char except newline)
|
|
classes [abc] [a-z] [^a-z] [a-c\d]
|
|
grouping ( ... ) (?: ... )
|
|
quantifiers * + ? {m} {m,} {m,n} (greedy only)
|
|
alternation | (empty branches allowed)
|
|
empty pattern / empty group "" ()
|
|
|
|
NOT SUPPORTED (raises RegexSyntaxError)
|
|
---------------------------------------
|
|
backreferences, lookaround, anchors, inline flags, lazy/possessive
|
|
quantifiers, named groups, \b \A \Z \n \t escapes.
|
|
|
|
SEMANTICS
|
|
---------
|
|
* fullmatch(p, s) -> bool exact language membership.
|
|
* match/search -> (start, end) LEFTMOST-LONGEST (POSIX-ish) match.
|
|
This differs from Python's leftmost-first backtracking: for `a|ab` on
|
|
"ab" this engine reports span (0, 2); `re` reports (0, 1). Only
|
|
`fullmatch` is compared against `re` in the fuzz harness, because
|
|
language membership is semantics-independent.
|
|
* Matching is O(len(pattern) * len(string)) with no backtracking, so
|
|
`(a|aa)*b` against "aaaa...a" is fast where `re` is exponential.
|
|
MEASURED on this host (Python 3.14.7, `--bench` output, answer is
|
|
False/None for every n):
|
|
n=22 re 0.0125s engine 0.00026s
|
|
n=26 re 0.1241s engine 0.00028s
|
|
n=30 re 1.4592s engine 0.00028s
|
|
n=34 re 9.7160s engine 0.01231s
|
|
n=60 re >10s (cut) engine 0.00042s
|
|
n=240 re >10s (cut) engine 0.00893s
|
|
(~10x per +4 chars for `re`; flat for the engine.)
|
|
Reproduce with: python3 regex_nfa.py --bench
|
|
|
|
CHECKS
|
|
------
|
|
python3 regex_nfa.py --check
|
|
runs (1) explicit unit tests, (2) a differential fuzz against `re` over
|
|
several seeds, and (3) a MUTATION test that injects 6 deliberate engine
|
|
bugs and requires the fuzzer to catch every one. Exit code 0 = all green.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import random
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
# --------------------------------------------------------------------------
|
|
# character sets
|
|
# --------------------------------------------------------------------------
|
|
|
|
WHITESPACE = frozenset(" \t\n\r\f\v")
|
|
DIGITS = frozenset("0123456789")
|
|
WORD = frozenset(
|
|
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
|
|
)
|
|
ESCAPABLE = ".^$*+?()[]{}|\\"
|
|
|
|
|
|
class RegexSyntaxError(ValueError):
|
|
"""Pattern outside the supported subset."""
|
|
|
|
|
|
def shorthand(e: str):
|
|
if e == "d":
|
|
return ("class", DIGITS, False)
|
|
if e == "D":
|
|
return ("class", DIGITS, True)
|
|
if e == "w":
|
|
return ("class", WORD, False)
|
|
if e == "W":
|
|
return ("class", WORD, True)
|
|
if e == "s":
|
|
return ("class", WHITESPACE, False)
|
|
if e == "S":
|
|
return ("class", WHITESPACE, True)
|
|
raise RegexSyntaxError("unsupported escape \\%s" % e)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# parser: pattern text -> raw AST
|
|
# AST nodes:
|
|
# ('empty',) ('char', c) ('any',)
|
|
# ('class', frozenset, neg) ('concat', [nodes]) ('alt', [nodes])
|
|
# ('repeat', node, m, n) n is None for unbounded
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
class Parser:
|
|
def __init__(self, pattern: str):
|
|
self.p = pattern
|
|
self.i = 0
|
|
|
|
def peek(self):
|
|
return self.p[self.i] if self.i < len(self.p) else None
|
|
|
|
def take(self):
|
|
ch = self.peek()
|
|
self.i += 1
|
|
return ch
|
|
|
|
def expect(self, ch):
|
|
if self.peek() != ch:
|
|
raise RegexSyntaxError(
|
|
"expected %r at %d in %r" % (ch, self.i, self.p)
|
|
)
|
|
self.i += 1
|
|
|
|
def parse(self):
|
|
node = self.parse_alt()
|
|
if self.i != len(self.p):
|
|
raise RegexSyntaxError("unexpected %r at %d" % (self.peek(), self.i))
|
|
return node
|
|
|
|
def parse_alt(self):
|
|
branches = [self.parse_concat()]
|
|
while self.peek() == "|":
|
|
self.i += 1
|
|
branches.append(self.parse_concat())
|
|
return branches[0] if len(branches) == 1 else ("alt", branches)
|
|
|
|
def parse_concat(self):
|
|
items = []
|
|
while True:
|
|
c = self.peek()
|
|
if c is None or c == "|" or c == ")":
|
|
break
|
|
items.append(self.parse_repeat())
|
|
if not items:
|
|
return ("empty",)
|
|
return items[0] if len(items) == 1 else ("concat", items)
|
|
|
|
def parse_repeat(self):
|
|
node = self.parse_atom()
|
|
seen = False
|
|
while True:
|
|
c = self.peek()
|
|
if c in ("*", "+", "?"):
|
|
if seen:
|
|
raise RegexSyntaxError("multiple repeat at %d" % self.i)
|
|
self.i += 1
|
|
seen = True
|
|
if c == "*":
|
|
node = ("repeat", node, 0, None)
|
|
elif c == "+":
|
|
node = ("repeat", node, 1, None)
|
|
else:
|
|
node = ("repeat", node, 0, 1)
|
|
elif c == "{":
|
|
if seen:
|
|
raise RegexSyntaxError("multiple repeat at %d" % self.i)
|
|
seen = True
|
|
m, n = self.parse_braces()
|
|
node = ("repeat", node, m, n)
|
|
else:
|
|
break
|
|
return node
|
|
|
|
def parse_int(self):
|
|
start = self.i
|
|
while self.peek() is not None and self.peek().isdigit():
|
|
self.i += 1
|
|
if self.i == start:
|
|
raise RegexSyntaxError("expected number at %d" % self.i)
|
|
return int(self.p[start:self.i])
|
|
|
|
def parse_braces(self):
|
|
self.expect("{")
|
|
m = self.parse_int()
|
|
if self.peek() == "}":
|
|
self.i += 1
|
|
return m, m
|
|
self.expect(",")
|
|
if self.peek() == "}":
|
|
self.i += 1
|
|
return m, None
|
|
n = self.parse_int()
|
|
self.expect("}")
|
|
if n < m:
|
|
raise RegexSyntaxError("repeat range %d,%d out of order" % (m, n))
|
|
return m, n
|
|
|
|
def parse_atom(self):
|
|
c = self.take()
|
|
if c is None:
|
|
raise RegexSyntaxError("unexpected end of pattern")
|
|
if c == "(":
|
|
if self.p.startswith("?:", self.i):
|
|
self.i += 2
|
|
node = self.parse_alt()
|
|
self.expect(")")
|
|
return node
|
|
if c == ")":
|
|
raise RegexSyntaxError("unmatched ')' at %d" % (self.i - 1))
|
|
if c == "[":
|
|
return self.parse_class()
|
|
if c == ".":
|
|
return ("any",)
|
|
if c == "\\":
|
|
e = self.take()
|
|
if e is None:
|
|
raise RegexSyntaxError("trailing backslash")
|
|
if e in "dDwWsS":
|
|
return shorthand(e)
|
|
if e in ESCAPABLE:
|
|
return ("char", e)
|
|
raise RegexSyntaxError("unsupported escape \\%s" % e)
|
|
if c in "*+?":
|
|
raise RegexSyntaxError("nothing to repeat at %d" % (self.i - 1))
|
|
return ("char", c)
|
|
|
|
def parse_class(self):
|
|
neg = False
|
|
if self.peek() == "^":
|
|
self.i += 1
|
|
neg = True
|
|
chars = set()
|
|
first = True
|
|
while True:
|
|
c = self.peek()
|
|
if c is None:
|
|
raise RegexSyntaxError("unterminated character set")
|
|
if c == "]":
|
|
if first:
|
|
# a leading ']' is a literal, as in `re`
|
|
self.i += 1
|
|
chars.add("]")
|
|
first = False
|
|
continue
|
|
self.i += 1
|
|
break
|
|
first = False
|
|
if c == "\\":
|
|
self.i += 1
|
|
e = self.take()
|
|
if e is None:
|
|
raise RegexSyntaxError("trailing backslash in class")
|
|
if e in "dws":
|
|
chars |= set(shorthand(e)[1])
|
|
continue
|
|
if e in "DWS":
|
|
raise RegexSyntaxError(
|
|
"\\%s inside a character class is not supported" % e
|
|
)
|
|
if e in ESCAPABLE:
|
|
start = e
|
|
else:
|
|
raise RegexSyntaxError("unsupported escape \\%s in class" % e)
|
|
else:
|
|
self.i += 1
|
|
start = c
|
|
if (
|
|
self.peek() == "-"
|
|
and self.i + 1 < len(self.p)
|
|
and self.p[self.i + 1] != "]"
|
|
):
|
|
self.i += 1 # consume '-'
|
|
end = self.take()
|
|
if end == "\\":
|
|
end = self.take()
|
|
if end not in ESCAPABLE:
|
|
raise RegexSyntaxError("bad range end")
|
|
if end is None or ord(end) < ord(start):
|
|
raise RegexSyntaxError("bad range %r-%r" % (start, end))
|
|
for o in range(ord(start), ord(end) + 1):
|
|
chars.add(chr(o))
|
|
else:
|
|
chars.add(start)
|
|
if not chars:
|
|
raise RegexSyntaxError("empty character class")
|
|
return ("class", frozenset(chars), neg)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# normalization: expand counted repetition into concat/alt/star
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def normalize(node):
|
|
k = node[0]
|
|
if k in ("empty", "char", "any", "class"):
|
|
return node
|
|
if k == "concat":
|
|
subs = [normalize(s) for s in node[1]]
|
|
subs = [s for s in subs if s[0] != "empty"]
|
|
if not subs:
|
|
return ("empty",)
|
|
return subs[0] if len(subs) == 1 else ("concat", subs)
|
|
if k == "alt":
|
|
return ("alt", [normalize(s) for s in node[1]])
|
|
if k == "star":
|
|
return ("star", normalize(node[1]))
|
|
if k == "repeat":
|
|
_, sub, m, n = node
|
|
sub = normalize(sub)
|
|
if n is None:
|
|
if m == 0:
|
|
return ("star", sub)
|
|
return ("concat", [sub] * m + [("star", sub)])
|
|
if n == 0:
|
|
return ("empty",)
|
|
if m == n:
|
|
return sub if m == 1 else ("concat", [sub] * m)
|
|
return ("concat", [sub] * m + [("alt", [("empty",), sub])] * (n - m))
|
|
raise RegexSyntaxError("bad node %r" % (node,))
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# NFA construction (Thompson)
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
class NFA:
|
|
def __init__(self):
|
|
self.eps = [] # state -> [state]
|
|
self.trans = [] # state -> [(predicate, state)]
|
|
|
|
def new_state(self):
|
|
self.eps.append([])
|
|
self.trans.append([])
|
|
return len(self.eps) - 1
|
|
|
|
|
|
def build(node, nfa):
|
|
k = node[0]
|
|
if k == "empty":
|
|
s, e = nfa.new_state(), nfa.new_state()
|
|
nfa.eps[s].append(e)
|
|
return s, e
|
|
if k in ("char", "any", "class"):
|
|
s, e = nfa.new_state(), nfa.new_state()
|
|
nfa.trans[s].append((node, e))
|
|
return s, e
|
|
if k == "concat":
|
|
s, e = build(node[1][0], nfa)
|
|
for sub in node[1][1:]:
|
|
s2, e2 = build(sub, nfa)
|
|
nfa.eps[e].append(s2)
|
|
e = e2
|
|
return s, e
|
|
if k == "alt":
|
|
s, e = nfa.new_state(), nfa.new_state()
|
|
for sub in node[1]:
|
|
s2, e2 = build(sub, nfa)
|
|
nfa.eps[s].append(s2)
|
|
nfa.eps[e2].append(e)
|
|
return s, e
|
|
if k == "star":
|
|
s, e = nfa.new_state(), nfa.new_state()
|
|
s2, e2 = build(node[1], nfa)
|
|
nfa.eps[s].append(e)
|
|
nfa.eps[s].append(s2)
|
|
nfa.eps[e2].append(s2)
|
|
nfa.eps[e2].append(e)
|
|
return s, e
|
|
raise RegexSyntaxError("bad node %r" % (node,))
|
|
|
|
|
|
def matches(pred, ch):
|
|
k = pred[0]
|
|
if k == "char":
|
|
return ch == pred[1]
|
|
if k == "any":
|
|
return ch != "\n"
|
|
return (ch not in pred[1]) if pred[2] else (ch in pred[1])
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# the engine
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
class Regex:
|
|
def __init__(self, pattern: str, mutate=None):
|
|
self.pattern = pattern
|
|
raw = Parser(pattern).parse()
|
|
if mutate:
|
|
raw = apply_mutation(raw, mutate)
|
|
self.ast = normalize(raw)
|
|
self.nfa = NFA()
|
|
self.start, self.accept = build(self.ast, self.nfa)
|
|
self.n_states = len(self.nfa.eps)
|
|
|
|
def __repr__(self):
|
|
return "Regex(%r, %d states)" % (self.pattern, self.n_states)
|
|
|
|
def _closure(self, states):
|
|
seen = set(states)
|
|
stack = list(states)
|
|
eps = self.nfa.eps
|
|
while stack:
|
|
s = stack.pop()
|
|
for t in eps[s]:
|
|
if t not in seen:
|
|
seen.add(t)
|
|
stack.append(t)
|
|
return seen
|
|
|
|
def _step(self, states, ch):
|
|
nxt = set()
|
|
for st in states:
|
|
for pred, tgt in self.nfa.trans[st]:
|
|
if matches(pred, ch):
|
|
nxt.add(tgt)
|
|
return self._closure(nxt) if nxt else set()
|
|
|
|
def fullmatch(self, s: str) -> bool:
|
|
cur = self._closure({self.start})
|
|
for ch in s:
|
|
if not cur:
|
|
return False
|
|
cur = self._step(cur, ch)
|
|
return self.accept in cur
|
|
|
|
def _longest_from(self, s, i):
|
|
cur = self._closure({self.start})
|
|
best = i if self.accept in cur else None
|
|
j = i
|
|
n = len(s)
|
|
while j < n and cur:
|
|
cur = self._step(cur, s[j])
|
|
j += 1
|
|
if self.accept in cur:
|
|
best = j
|
|
return best
|
|
|
|
def match(self, s):
|
|
e = self._longest_from(s, 0)
|
|
return (0, e) if e is not None else None
|
|
|
|
def search(self, s):
|
|
for i in range(len(s) + 1):
|
|
e = self._longest_from(s, i)
|
|
if e is not None:
|
|
return (i, e)
|
|
return None
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# mutation harness: deliberately broken engines, all of which MUST be caught
|
|
# by the differential fuzzer (this is what makes --check non-tautological)
|
|
# --------------------------------------------------------------------------
|
|
|
|
MUTATIONS = {
|
|
"star_empty": "x* -> empty (drops repetition)",
|
|
"alt_first": "x|y -> x (drops later alternatives)",
|
|
"plus_to_star": "x+ -> x* (requires nothing)",
|
|
"any_all": ". -> any char including newline",
|
|
"neg_class": "[^x] -> [x] (drops class negation)",
|
|
"opt_required": "x? -> x (makes an optional mandatory)",
|
|
}
|
|
|
|
|
|
def apply_mutation(node, mode):
|
|
k = node[0]
|
|
if k == "repeat":
|
|
sub = apply_mutation(node[1], mode)
|
|
m, n = node[2], node[3]
|
|
if mode == "star_empty" and m == 0 and n is None:
|
|
return ("empty",)
|
|
if mode == "plus_to_star" and m == 1 and n is None:
|
|
return ("repeat", sub, 0, None)
|
|
if mode == "opt_required" and m == 0 and n == 1:
|
|
return sub
|
|
return ("repeat", sub, m, n)
|
|
if k == "alt":
|
|
subs = [apply_mutation(s, mode) for s in node[1]]
|
|
return subs[0] if mode == "alt_first" else ("alt", subs)
|
|
if k == "concat":
|
|
return ("concat", [apply_mutation(s, mode) for s in node[1]])
|
|
if k == "any":
|
|
return ("class", frozenset(), True) if mode == "any_all" else node
|
|
if k == "class":
|
|
if mode == "neg_class" and node[2]:
|
|
return ("class", node[1], False)
|
|
return node
|
|
return node
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# differential fuzzer: random patterns from the shared subset, compared to `re`
|
|
# --------------------------------------------------------------------------
|
|
|
|
ALPHA = "abc01_ "
|
|
SPECIALS = ".^$*+?()[]{}|\\"
|
|
|
|
|
|
def gen_atom(rng, depth=0):
|
|
r = rng.random()
|
|
if r < 0.30:
|
|
return rng.choice(ALPHA)
|
|
if r < 0.44:
|
|
return "\\" + rng.choice(SPECIALS)
|
|
if r < 0.54:
|
|
return "."
|
|
if r < 0.69:
|
|
return rng.choice(["\\d", "\\w", "\\s", "\\D", "\\W", "\\S"])
|
|
if r < 0.89:
|
|
return gen_class(rng)
|
|
return "(" + gen_pattern(rng, depth + 1) + ")"
|
|
|
|
|
|
def gen_class(rng):
|
|
neg = "^" if rng.random() < 0.35 else ""
|
|
items = []
|
|
for _ in range(rng.randint(1, 3)):
|
|
r = rng.random()
|
|
if r < 0.50:
|
|
items.append(rng.choice(ALPHA))
|
|
elif r < 0.80:
|
|
a, b = sorted(rng.sample("abc012", 2))
|
|
items.append("%s-%s" % (a, b))
|
|
else:
|
|
items.append(rng.choice(["\\d", "\\w", "\\s"]))
|
|
return "[" + neg + "".join(items) + "]"
|
|
|
|
|
|
def gen_pattern(rng, depth=0):
|
|
r = rng.random()
|
|
if depth >= 3:
|
|
return gen_atom(rng, depth)
|
|
if r < 0.30:
|
|
return "".join(gen_atom(rng, depth + 1) for _ in range(rng.randint(2, 3)))
|
|
if r < 0.50:
|
|
n = rng.randint(2, 3)
|
|
branches = [gen_pattern(rng, depth + 1) for _ in range(n)]
|
|
if rng.random() < 0.20:
|
|
branches[rng.randrange(n)] = ""
|
|
return "(" + "|".join(branches) + ")"
|
|
if r < 0.80:
|
|
return gen_atom(rng, depth + 1) + rng.choice(
|
|
["*", "+", "?", "{2}", "{1,2}", "{0,3}", "{2,}"]
|
|
)
|
|
return gen_atom(rng, depth + 1)
|
|
|
|
|
|
def gen_string(rng, maxlen=8):
|
|
out = []
|
|
for _ in range(rng.randint(0, maxlen)):
|
|
out.append("\n" if rng.random() < 0.12 else rng.choice(ALPHA))
|
|
return "".join(out)
|
|
|
|
|
|
def fuzz(n, seed, mutate=None):
|
|
"""Return (mismatches, skips). mismatches = engine disagrees with `re`."""
|
|
rng = random.Random(seed)
|
|
mismatches = []
|
|
skips = 0
|
|
for _ in range(n):
|
|
p = gen_pattern(rng)
|
|
s = gen_string(rng)
|
|
try:
|
|
theirs = re.fullmatch(p, s) is not None
|
|
except re.error:
|
|
skips += 1
|
|
continue
|
|
try:
|
|
mine = Regex(p, mutate=mutate).fullmatch(s)
|
|
except RegexSyntaxError as exc:
|
|
mismatches.append((p, s, "engine-error: %s" % exc, theirs))
|
|
continue
|
|
if mine != theirs:
|
|
mismatches.append((p, s, mine, theirs))
|
|
if len(mismatches) >= 5:
|
|
break
|
|
return mismatches, skips
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# checks
|
|
# --------------------------------------------------------------------------
|
|
|
|
UNIT_TESTS = [
|
|
# pattern, string, fullmatch expected
|
|
("a", "a", True),
|
|
("a", "b", False),
|
|
("a", "", False),
|
|
("", "", True),
|
|
("a*", "", True),
|
|
("a*", "aaaa", True),
|
|
("a*", "aab", False),
|
|
("(a|b)*", "abba", True),
|
|
("(a|b)*", "abbc", False),
|
|
("(ab)+", "abab", True),
|
|
("(ab)+", "aba", False),
|
|
("a{2,3}", "aa", True),
|
|
("a{2,3}", "aaa", True),
|
|
("a{2,3}", "a", False),
|
|
("a{2,3}", "aaaa", False),
|
|
("a{2}", "aa", True),
|
|
("a{2,}", "aaaa", True),
|
|
("a{2,}", "a", False),
|
|
("a|", "", True),
|
|
("a|", "a", True),
|
|
("(a|)b", "b", True),
|
|
("[^a]", "b", True),
|
|
("[^a]", "a", False),
|
|
("[^a]", "\n", True), # negated classes do match newline in `re` too
|
|
(".", "x", True),
|
|
(".", "\n", False), # dot does not match newline
|
|
("\\d+", "123", True),
|
|
("\\d+", "12a", False),
|
|
("\\w+", "ab_1", True),
|
|
("\\w+", "a b", False),
|
|
("\\s", " ", True),
|
|
("\\s", "\n", True),
|
|
("\\S", "x", True),
|
|
("\\S", " ", False),
|
|
("\\D", "a", True),
|
|
("\\D", "7", False),
|
|
("[a-c0-2]+", "ab0c2", True),
|
|
("[a-c0-2]+", "abz", False),
|
|
("[a-c\\d]+", "ab7", True),
|
|
("[a-c\\d]+", "abz", False),
|
|
("(?:ab)+", "abab", True),
|
|
("\\.", ".", True),
|
|
("\\.", "x", False),
|
|
("\\*", "*", True),
|
|
("a\\.b", "a.b", True),
|
|
("[\\]]", "]", True),
|
|
]
|
|
|
|
SPAN_TESTS = [
|
|
# pattern, string, expected (start, end) under LEFTMOST-LONGEST
|
|
("a+", "baaa", (1, 4)),
|
|
("a*", "b", (0, 0)),
|
|
("a+", "bbb", None),
|
|
("[0-9]+", "ab123cd", (2, 5)),
|
|
# documented divergence from Python's leftmost-FIRST `re` (which gives (0,1)):
|
|
("a|ab", "ab", (0, 2)),
|
|
]
|
|
|
|
|
|
def run_check(verbose=True):
|
|
failures = []
|
|
total = 0
|
|
|
|
# (1) unit tests
|
|
for p, s, want in UNIT_TESTS:
|
|
total += 1
|
|
got = Regex(p).fullmatch(s)
|
|
if got != want:
|
|
failures.append("fullmatch(%r, %r) = %r, want %r" % (p, s, got, want))
|
|
if verbose and got != want:
|
|
print(" FAIL unit: fullmatch(%r, %r) = %r, want %r" % (p, s, got, want))
|
|
|
|
for p, s, want in SPAN_TESTS:
|
|
total += 1
|
|
got = Regex(p).search(s)
|
|
if got != want:
|
|
failures.append("search(%r, %r) = %r, want %r" % (p, s, got, want))
|
|
if verbose:
|
|
print(" FAIL span: search(%r, %r) = %r, want %r" % (p, s, got, want))
|
|
|
|
# (2) no-backtracking / pathological-case check
|
|
total += 1
|
|
t0 = time.perf_counter()
|
|
got = Regex("(a|aa)*b").fullmatch("a" * 40)
|
|
dt = time.perf_counter() - t0
|
|
if got is not False or dt > 2.0:
|
|
failures.append("pathological (a|aa)*b on a*40 -> %r in %.3fs" % (got, dt))
|
|
else:
|
|
print(" pathological (a|aa)*b vs 40 a's: %r in %.4fs (re is exponential here)"
|
|
% (got, dt))
|
|
|
|
# (3) differential fuzz against `re`, several seeds
|
|
fuzz_total = 0
|
|
for seed in (1, 7, 20261021, 99991):
|
|
mm, skips = fuzz(1200, seed)
|
|
fuzz_total += 1200
|
|
total += 1200
|
|
if mm:
|
|
failures.append("fuzz seed=%d mismatches: %r" % (seed, mm[:3]))
|
|
if verbose:
|
|
print(" FAIL fuzz seed=%d: %r" % (seed, mm[:3]))
|
|
elif verbose:
|
|
print(" fuzz seed=%-8d 1200 cases: 0 mismatches (%d unparseable-by-re)"
|
|
% (seed, skips))
|
|
|
|
# (4) mutation test: every injected bug must be caught by the fuzzer
|
|
caught = {}
|
|
for mode in sorted(MUTATIONS):
|
|
mm, _ = fuzz(600, 4242, mutate=mode)
|
|
caught[mode] = bool(mm)
|
|
total += 1
|
|
if not mm:
|
|
failures.append("mutant %r survived the fuzzer" % mode)
|
|
if verbose:
|
|
print(" FAIL mutant survived: %s (%s)" % (mode, MUTATIONS[mode]))
|
|
elif verbose:
|
|
print(" mutant %-14s caught after fuzz (%s)"
|
|
% (mode, MUTATIONS[mode]))
|
|
|
|
print("-" * 72)
|
|
if failures:
|
|
print("FAILED: %d problem(s) out of %d checks" % (len(failures), total))
|
|
for f in failures[:10]:
|
|
print(" - %s" % f)
|
|
return 1
|
|
print("OK: %d checks green (%d differential-fuzz cases vs `re`, "
|
|
"%d/%d injected mutants caught)"
|
|
% (total, fuzz_total, sum(caught.values()), len(caught)))
|
|
return 0
|
|
|
|
|
|
def _timed(fn, budget=10.0):
|
|
"""Run fn() under a SIGALRM budget. Returns (seconds, result) or (None, None)."""
|
|
import signal
|
|
|
|
def handler(signum, frame):
|
|
raise TimeoutError()
|
|
|
|
old = signal.signal(signal.SIGALRM, handler)
|
|
signal.setitimer(signal.ITIMER_REAL, budget)
|
|
t0 = time.perf_counter()
|
|
try:
|
|
res = fn()
|
|
return time.perf_counter() - t0, res
|
|
except TimeoutError:
|
|
return None, None
|
|
finally:
|
|
signal.setitimer(signal.ITIMER_REAL, 0)
|
|
signal.signal(signal.SIGALRM, old)
|
|
|
|
|
|
def bench(budget=10.0):
|
|
"""Show that `re` backtracks exponentially where this engine does not."""
|
|
print("pattern '(a|aa)*b' against 'a'*n (answer is False/None for every n)")
|
|
print(" %-6s %-14s %-14s" % ("n", "re.fullmatch", "Regex.fullmatch"))
|
|
for n in (22, 26, 30, 34, 60, 240):
|
|
s = "a" * n
|
|
t_re, r_re = _timed(lambda: re.fullmatch("(a|aa)*b", s), budget)
|
|
t_nfa, r_nfa = _timed(lambda: Regex("(a|aa)*b").fullmatch(s), budget)
|
|
print(" %-6d %-14s %-14s re=%r engine=%r"
|
|
% (n,
|
|
">%.0fs (cut)" % budget if t_re is None else "%.4fs" % t_re,
|
|
">%.0fs (cut)" % budget if t_nfa is None else "%.5fs" % t_nfa,
|
|
r_re, r_nfa))
|
|
return 0
|
|
|
|
|
|
def demo():
|
|
for p in ["a(b|c)*d", "(a|aa)*b", "[^0-9]{2,3}", "\\w+@\\w+", "(a|)b"]:
|
|
r = Regex(p)
|
|
print("%-14r %3d NFA states" % (p, r.n_states))
|
|
for s in ["abcd", "abbcd", "aaa", "x7", "b", ""]:
|
|
print(" fullmatch(%-8r) = %-5s search -> %r"
|
|
% (s, r.fullmatch(s), r.search(s)))
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description="Thompson-NFA regex engine")
|
|
ap.add_argument("--check", action="store_true",
|
|
help="run unit tests + differential fuzz + mutation test")
|
|
ap.add_argument("--demo", action="store_true", help="print a small demo")
|
|
ap.add_argument("--bench", action="store_true",
|
|
help="measure re's backtracking blowup vs this engine")
|
|
ap.add_argument("-p", "--pattern", help="pattern to match")
|
|
ap.add_argument("-s", "--string", default="", help="subject string")
|
|
ap.add_argument("--fuzz", type=int, help="ad-hoc fuzz case count")
|
|
ap.add_argument("--seed", type=int, default=20261021, help="fuzz seed")
|
|
args = ap.parse_args(argv)
|
|
|
|
if args.check:
|
|
return run_check()
|
|
if args.demo:
|
|
demo()
|
|
return 0
|
|
if args.bench:
|
|
return bench()
|
|
if args.fuzz is not None:
|
|
mm, skips = fuzz(args.fuzz, args.seed)
|
|
print("cases=%d mismatches=%d unparseable-by-re=%d" % (args.fuzz, len(mm), skips))
|
|
for p, s, mine, theirs in mm[:10]:
|
|
print(" %r on %r: engine=%r re=%r" % (p, s, mine, theirs))
|
|
return 1 if mm else 0
|
|
if args.pattern is not None:
|
|
r = Regex(args.pattern)
|
|
print("pattern : %r (%d NFA states)" % (args.pattern, r.n_states))
|
|
print("fullmatch : %s" % r.fullmatch(args.string))
|
|
print("match : %r" % (r.match(args.string),))
|
|
print("search : %r" % (r.search(args.string),))
|
|
return 0
|
|
ap.print_help()
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|