#!/usr/bin/env python3 r""" regex_nfa.py -- a Thompson-NFA regular-expression engine, stdlib only. WHY --- This is a from-scratch regex engine used as a *differential test target*: every pattern/string pair it evaluates is compared against Python's own `re` module for language membership (`fullmatch`). If the two ever disagree on a pattern inside the supported subset, one of them is wrong. SUPPORTED SUBSET (exactly what `re` accepts here too) ----------------------------------------------------- literals, escaped metacharacters \. \\ \* \+ \? \( \) \[ \] \| \{ \} shorthand classes \d \w \s \D \W \S dot . (any char except newline) classes [abc] [a-z] [^a-z] [a-c\d] grouping ( ... ) (?: ... ) quantifiers * + ? {m} {m,} {m,n} (greedy only) alternation | (empty branches allowed) empty pattern / empty group "" () NOT SUPPORTED (raises RegexSyntaxError) --------------------------------------- backreferences, lookaround, anchors, inline flags, lazy/possessive quantifiers, named groups, \b \A \Z \n \t escapes. SEMANTICS --------- * fullmatch(p, s) -> bool exact language membership. * match/search -> (start, end) LEFTMOST-LONGEST (POSIX-ish) match. This differs from Python's leftmost-first backtracking: for `a|ab` on "ab" this engine reports span (0, 2); `re` reports (0, 1). Only `fullmatch` is compared against `re` in the fuzz harness, because language membership is semantics-independent. * Matching is O(len(pattern) * len(string)) with no backtracking, so `(a|aa)*b` against "aaaa...a" is fast where `re` is exponential. MEASURED on this host (Python 3.14.7, `--bench` output, answer is False/None for every n): n=22 re 0.0125s engine 0.00026s n=26 re 0.1241s engine 0.00028s n=30 re 1.4592s engine 0.00028s n=34 re 9.7160s engine 0.01231s n=60 re >10s (cut) engine 0.00042s n=240 re >10s (cut) engine 0.00893s (~10x per +4 chars for `re`; flat for the engine.) Reproduce with: python3 regex_nfa.py --bench CHECKS ------ python3 regex_nfa.py --check runs (1) explicit unit tests, (2) a differential fuzz against `re` over several seeds, and (3) a MUTATION test that injects 6 deliberate engine bugs and requires the fuzzer to catch every one. Exit code 0 = all green. """ from __future__ import annotations import argparse import random import re import sys import time # -------------------------------------------------------------------------- # character sets # -------------------------------------------------------------------------- WHITESPACE = frozenset(" \t\n\r\f\v") DIGITS = frozenset("0123456789") WORD = frozenset( "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" ) ESCAPABLE = ".^$*+?()[]{}|\\" class RegexSyntaxError(ValueError): """Pattern outside the supported subset.""" def shorthand(e: str): if e == "d": return ("class", DIGITS, False) if e == "D": return ("class", DIGITS, True) if e == "w": return ("class", WORD, False) if e == "W": return ("class", WORD, True) if e == "s": return ("class", WHITESPACE, False) if e == "S": return ("class", WHITESPACE, True) raise RegexSyntaxError("unsupported escape \\%s" % e) # -------------------------------------------------------------------------- # parser: pattern text -> raw AST # AST nodes: # ('empty',) ('char', c) ('any',) # ('class', frozenset, neg) ('concat', [nodes]) ('alt', [nodes]) # ('repeat', node, m, n) n is None for unbounded # -------------------------------------------------------------------------- class Parser: def __init__(self, pattern: str): self.p = pattern self.i = 0 def peek(self): return self.p[self.i] if self.i < len(self.p) else None def take(self): ch = self.peek() self.i += 1 return ch def expect(self, ch): if self.peek() != ch: raise RegexSyntaxError( "expected %r at %d in %r" % (ch, self.i, self.p) ) self.i += 1 def parse(self): node = self.parse_alt() if self.i != len(self.p): raise RegexSyntaxError("unexpected %r at %d" % (self.peek(), self.i)) return node def parse_alt(self): branches = [self.parse_concat()] while self.peek() == "|": self.i += 1 branches.append(self.parse_concat()) return branches[0] if len(branches) == 1 else ("alt", branches) def parse_concat(self): items = [] while True: c = self.peek() if c is None or c == "|" or c == ")": break items.append(self.parse_repeat()) if not items: return ("empty",) return items[0] if len(items) == 1 else ("concat", items) def parse_repeat(self): node = self.parse_atom() seen = False while True: c = self.peek() if c in ("*", "+", "?"): if seen: raise RegexSyntaxError("multiple repeat at %d" % self.i) self.i += 1 seen = True if c == "*": node = ("repeat", node, 0, None) elif c == "+": node = ("repeat", node, 1, None) else: node = ("repeat", node, 0, 1) elif c == "{": if seen: raise RegexSyntaxError("multiple repeat at %d" % self.i) seen = True m, n = self.parse_braces() node = ("repeat", node, m, n) else: break return node def parse_int(self): start = self.i while self.peek() is not None and self.peek().isdigit(): self.i += 1 if self.i == start: raise RegexSyntaxError("expected number at %d" % self.i) return int(self.p[start:self.i]) def parse_braces(self): self.expect("{") m = self.parse_int() if self.peek() == "}": self.i += 1 return m, m self.expect(",") if self.peek() == "}": self.i += 1 return m, None n = self.parse_int() self.expect("}") if n < m: raise RegexSyntaxError("repeat range %d,%d out of order" % (m, n)) return m, n def parse_atom(self): c = self.take() if c is None: raise RegexSyntaxError("unexpected end of pattern") if c == "(": if self.p.startswith("?:", self.i): self.i += 2 node = self.parse_alt() self.expect(")") return node if c == ")": raise RegexSyntaxError("unmatched ')' at %d" % (self.i - 1)) if c == "[": return self.parse_class() if c == ".": return ("any",) if c == "\\": e = self.take() if e is None: raise RegexSyntaxError("trailing backslash") if e in "dDwWsS": return shorthand(e) if e in ESCAPABLE: return ("char", e) raise RegexSyntaxError("unsupported escape \\%s" % e) if c in "*+?": raise RegexSyntaxError("nothing to repeat at %d" % (self.i - 1)) return ("char", c) def parse_class(self): neg = False if self.peek() == "^": self.i += 1 neg = True chars = set() first = True while True: c = self.peek() if c is None: raise RegexSyntaxError("unterminated character set") if c == "]": if first: # a leading ']' is a literal, as in `re` self.i += 1 chars.add("]") first = False continue self.i += 1 break first = False if c == "\\": self.i += 1 e = self.take() if e is None: raise RegexSyntaxError("trailing backslash in class") if e in "dws": chars |= set(shorthand(e)[1]) continue if e in "DWS": raise RegexSyntaxError( "\\%s inside a character class is not supported" % e ) if e in ESCAPABLE: start = e else: raise RegexSyntaxError("unsupported escape \\%s in class" % e) else: self.i += 1 start = c if ( self.peek() == "-" and self.i + 1 < len(self.p) and self.p[self.i + 1] != "]" ): self.i += 1 # consume '-' end = self.take() if end == "\\": end = self.take() if end not in ESCAPABLE: raise RegexSyntaxError("bad range end") if end is None or ord(end) < ord(start): raise RegexSyntaxError("bad range %r-%r" % (start, end)) for o in range(ord(start), ord(end) + 1): chars.add(chr(o)) else: chars.add(start) if not chars: raise RegexSyntaxError("empty character class") return ("class", frozenset(chars), neg) # -------------------------------------------------------------------------- # normalization: expand counted repetition into concat/alt/star # -------------------------------------------------------------------------- def normalize(node): k = node[0] if k in ("empty", "char", "any", "class"): return node if k == "concat": subs = [normalize(s) for s in node[1]] subs = [s for s in subs if s[0] != "empty"] if not subs: return ("empty",) return subs[0] if len(subs) == 1 else ("concat", subs) if k == "alt": return ("alt", [normalize(s) for s in node[1]]) if k == "star": return ("star", normalize(node[1])) if k == "repeat": _, sub, m, n = node sub = normalize(sub) if n is None: if m == 0: return ("star", sub) return ("concat", [sub] * m + [("star", sub)]) if n == 0: return ("empty",) if m == n: return sub if m == 1 else ("concat", [sub] * m) return ("concat", [sub] * m + [("alt", [("empty",), sub])] * (n - m)) raise RegexSyntaxError("bad node %r" % (node,)) # -------------------------------------------------------------------------- # NFA construction (Thompson) # -------------------------------------------------------------------------- class NFA: def __init__(self): self.eps = [] # state -> [state] self.trans = [] # state -> [(predicate, state)] def new_state(self): self.eps.append([]) self.trans.append([]) return len(self.eps) - 1 def build(node, nfa): k = node[0] if k == "empty": s, e = nfa.new_state(), nfa.new_state() nfa.eps[s].append(e) return s, e if k in ("char", "any", "class"): s, e = nfa.new_state(), nfa.new_state() nfa.trans[s].append((node, e)) return s, e if k == "concat": s, e = build(node[1][0], nfa) for sub in node[1][1:]: s2, e2 = build(sub, nfa) nfa.eps[e].append(s2) e = e2 return s, e if k == "alt": s, e = nfa.new_state(), nfa.new_state() for sub in node[1]: s2, e2 = build(sub, nfa) nfa.eps[s].append(s2) nfa.eps[e2].append(e) return s, e if k == "star": s, e = nfa.new_state(), nfa.new_state() s2, e2 = build(node[1], nfa) nfa.eps[s].append(e) nfa.eps[s].append(s2) nfa.eps[e2].append(s2) nfa.eps[e2].append(e) return s, e raise RegexSyntaxError("bad node %r" % (node,)) def matches(pred, ch): k = pred[0] if k == "char": return ch == pred[1] if k == "any": return ch != "\n" return (ch not in pred[1]) if pred[2] else (ch in pred[1]) # -------------------------------------------------------------------------- # the engine # -------------------------------------------------------------------------- class Regex: def __init__(self, pattern: str, mutate=None): self.pattern = pattern raw = Parser(pattern).parse() if mutate: raw = apply_mutation(raw, mutate) self.ast = normalize(raw) self.nfa = NFA() self.start, self.accept = build(self.ast, self.nfa) self.n_states = len(self.nfa.eps) def __repr__(self): return "Regex(%r, %d states)" % (self.pattern, self.n_states) def _closure(self, states): seen = set(states) stack = list(states) eps = self.nfa.eps while stack: s = stack.pop() for t in eps[s]: if t not in seen: seen.add(t) stack.append(t) return seen def _step(self, states, ch): nxt = set() for st in states: for pred, tgt in self.nfa.trans[st]: if matches(pred, ch): nxt.add(tgt) return self._closure(nxt) if nxt else set() def fullmatch(self, s: str) -> bool: cur = self._closure({self.start}) for ch in s: if not cur: return False cur = self._step(cur, ch) return self.accept in cur def _longest_from(self, s, i): cur = self._closure({self.start}) best = i if self.accept in cur else None j = i n = len(s) while j < n and cur: cur = self._step(cur, s[j]) j += 1 if self.accept in cur: best = j return best def match(self, s): e = self._longest_from(s, 0) return (0, e) if e is not None else None def search(self, s): for i in range(len(s) + 1): e = self._longest_from(s, i) if e is not None: return (i, e) return None # -------------------------------------------------------------------------- # mutation harness: deliberately broken engines, all of which MUST be caught # by the differential fuzzer (this is what makes --check non-tautological) # -------------------------------------------------------------------------- MUTATIONS = { "star_empty": "x* -> empty (drops repetition)", "alt_first": "x|y -> x (drops later alternatives)", "plus_to_star": "x+ -> x* (requires nothing)", "any_all": ". -> any char including newline", "neg_class": "[^x] -> [x] (drops class negation)", "opt_required": "x? -> x (makes an optional mandatory)", } def apply_mutation(node, mode): k = node[0] if k == "repeat": sub = apply_mutation(node[1], mode) m, n = node[2], node[3] if mode == "star_empty" and m == 0 and n is None: return ("empty",) if mode == "plus_to_star" and m == 1 and n is None: return ("repeat", sub, 0, None) if mode == "opt_required" and m == 0 and n == 1: return sub return ("repeat", sub, m, n) if k == "alt": subs = [apply_mutation(s, mode) for s in node[1]] return subs[0] if mode == "alt_first" else ("alt", subs) if k == "concat": return ("concat", [apply_mutation(s, mode) for s in node[1]]) if k == "any": return ("class", frozenset(), True) if mode == "any_all" else node if k == "class": if mode == "neg_class" and node[2]: return ("class", node[1], False) return node return node # -------------------------------------------------------------------------- # differential fuzzer: random patterns from the shared subset, compared to `re` # -------------------------------------------------------------------------- ALPHA = "abc01_ " SPECIALS = ".^$*+?()[]{}|\\" def gen_atom(rng, depth=0): r = rng.random() if r < 0.30: return rng.choice(ALPHA) if r < 0.44: return "\\" + rng.choice(SPECIALS) if r < 0.54: return "." if r < 0.69: return rng.choice(["\\d", "\\w", "\\s", "\\D", "\\W", "\\S"]) if r < 0.89: return gen_class(rng) return "(" + gen_pattern(rng, depth + 1) + ")" def gen_class(rng): neg = "^" if rng.random() < 0.35 else "" items = [] for _ in range(rng.randint(1, 3)): r = rng.random() if r < 0.50: items.append(rng.choice(ALPHA)) elif r < 0.80: a, b = sorted(rng.sample("abc012", 2)) items.append("%s-%s" % (a, b)) else: items.append(rng.choice(["\\d", "\\w", "\\s"])) return "[" + neg + "".join(items) + "]" def gen_pattern(rng, depth=0): r = rng.random() if depth >= 3: return gen_atom(rng, depth) if r < 0.30: return "".join(gen_atom(rng, depth + 1) for _ in range(rng.randint(2, 3))) if r < 0.50: n = rng.randint(2, 3) branches = [gen_pattern(rng, depth + 1) for _ in range(n)] if rng.random() < 0.20: branches[rng.randrange(n)] = "" return "(" + "|".join(branches) + ")" if r < 0.80: return gen_atom(rng, depth + 1) + rng.choice( ["*", "+", "?", "{2}", "{1,2}", "{0,3}", "{2,}"] ) return gen_atom(rng, depth + 1) def gen_string(rng, maxlen=8): out = [] for _ in range(rng.randint(0, maxlen)): out.append("\n" if rng.random() < 0.12 else rng.choice(ALPHA)) return "".join(out) def fuzz(n, seed, mutate=None): """Return (mismatches, skips). mismatches = engine disagrees with `re`.""" rng = random.Random(seed) mismatches = [] skips = 0 for _ in range(n): p = gen_pattern(rng) s = gen_string(rng) try: theirs = re.fullmatch(p, s) is not None except re.error: skips += 1 continue try: mine = Regex(p, mutate=mutate).fullmatch(s) except RegexSyntaxError as exc: mismatches.append((p, s, "engine-error: %s" % exc, theirs)) continue if mine != theirs: mismatches.append((p, s, mine, theirs)) if len(mismatches) >= 5: break return mismatches, skips # -------------------------------------------------------------------------- # checks # -------------------------------------------------------------------------- UNIT_TESTS = [ # pattern, string, fullmatch expected ("a", "a", True), ("a", "b", False), ("a", "", False), ("", "", True), ("a*", "", True), ("a*", "aaaa", True), ("a*", "aab", False), ("(a|b)*", "abba", True), ("(a|b)*", "abbc", False), ("(ab)+", "abab", True), ("(ab)+", "aba", False), ("a{2,3}", "aa", True), ("a{2,3}", "aaa", True), ("a{2,3}", "a", False), ("a{2,3}", "aaaa", False), ("a{2}", "aa", True), ("a{2,}", "aaaa", True), ("a{2,}", "a", False), ("a|", "", True), ("a|", "a", True), ("(a|)b", "b", True), ("[^a]", "b", True), ("[^a]", "a", False), ("[^a]", "\n", True), # negated classes do match newline in `re` too (".", "x", True), (".", "\n", False), # dot does not match newline ("\\d+", "123", True), ("\\d+", "12a", False), ("\\w+", "ab_1", True), ("\\w+", "a b", False), ("\\s", " ", True), ("\\s", "\n", True), ("\\S", "x", True), ("\\S", " ", False), ("\\D", "a", True), ("\\D", "7", False), ("[a-c0-2]+", "ab0c2", True), ("[a-c0-2]+", "abz", False), ("[a-c\\d]+", "ab7", True), ("[a-c\\d]+", "abz", False), ("(?:ab)+", "abab", True), ("\\.", ".", True), ("\\.", "x", False), ("\\*", "*", True), ("a\\.b", "a.b", True), ("[\\]]", "]", True), ] SPAN_TESTS = [ # pattern, string, expected (start, end) under LEFTMOST-LONGEST ("a+", "baaa", (1, 4)), ("a*", "b", (0, 0)), ("a+", "bbb", None), ("[0-9]+", "ab123cd", (2, 5)), # documented divergence from Python's leftmost-FIRST `re` (which gives (0,1)): ("a|ab", "ab", (0, 2)), ] def run_check(verbose=True): failures = [] total = 0 # (1) unit tests for p, s, want in UNIT_TESTS: total += 1 got = Regex(p).fullmatch(s) if got != want: failures.append("fullmatch(%r, %r) = %r, want %r" % (p, s, got, want)) if verbose and got != want: print(" FAIL unit: fullmatch(%r, %r) = %r, want %r" % (p, s, got, want)) for p, s, want in SPAN_TESTS: total += 1 got = Regex(p).search(s) if got != want: failures.append("search(%r, %r) = %r, want %r" % (p, s, got, want)) if verbose: print(" FAIL span: search(%r, %r) = %r, want %r" % (p, s, got, want)) # (2) no-backtracking / pathological-case check total += 1 t0 = time.perf_counter() got = Regex("(a|aa)*b").fullmatch("a" * 40) dt = time.perf_counter() - t0 if got is not False or dt > 2.0: failures.append("pathological (a|aa)*b on a*40 -> %r in %.3fs" % (got, dt)) else: print(" pathological (a|aa)*b vs 40 a's: %r in %.4fs (re is exponential here)" % (got, dt)) # (3) differential fuzz against `re`, several seeds fuzz_total = 0 for seed in (1, 7, 20261021, 99991): mm, skips = fuzz(1200, seed) fuzz_total += 1200 total += 1200 if mm: failures.append("fuzz seed=%d mismatches: %r" % (seed, mm[:3])) if verbose: print(" FAIL fuzz seed=%d: %r" % (seed, mm[:3])) elif verbose: print(" fuzz seed=%-8d 1200 cases: 0 mismatches (%d unparseable-by-re)" % (seed, skips)) # (4) mutation test: every injected bug must be caught by the fuzzer caught = {} for mode in sorted(MUTATIONS): mm, _ = fuzz(600, 4242, mutate=mode) caught[mode] = bool(mm) total += 1 if not mm: failures.append("mutant %r survived the fuzzer" % mode) if verbose: print(" FAIL mutant survived: %s (%s)" % (mode, MUTATIONS[mode])) elif verbose: print(" mutant %-14s caught after fuzz (%s)" % (mode, MUTATIONS[mode])) print("-" * 72) if failures: print("FAILED: %d problem(s) out of %d checks" % (len(failures), total)) for f in failures[:10]: print(" - %s" % f) return 1 print("OK: %d checks green (%d differential-fuzz cases vs `re`, " "%d/%d injected mutants caught)" % (total, fuzz_total, sum(caught.values()), len(caught))) return 0 def _timed(fn, budget=10.0): """Run fn() under a SIGALRM budget. Returns (seconds, result) or (None, None).""" import signal def handler(signum, frame): raise TimeoutError() old = signal.signal(signal.SIGALRM, handler) signal.setitimer(signal.ITIMER_REAL, budget) t0 = time.perf_counter() try: res = fn() return time.perf_counter() - t0, res except TimeoutError: return None, None finally: signal.setitimer(signal.ITIMER_REAL, 0) signal.signal(signal.SIGALRM, old) def bench(budget=10.0): """Show that `re` backtracks exponentially where this engine does not.""" print("pattern '(a|aa)*b' against 'a'*n (answer is False/None for every n)") print(" %-6s %-14s %-14s" % ("n", "re.fullmatch", "Regex.fullmatch")) for n in (22, 26, 30, 34, 60, 240): s = "a" * n t_re, r_re = _timed(lambda: re.fullmatch("(a|aa)*b", s), budget) t_nfa, r_nfa = _timed(lambda: Regex("(a|aa)*b").fullmatch(s), budget) print(" %-6d %-14s %-14s re=%r engine=%r" % (n, ">%.0fs (cut)" % budget if t_re is None else "%.4fs" % t_re, ">%.0fs (cut)" % budget if t_nfa is None else "%.5fs" % t_nfa, r_re, r_nfa)) return 0 def demo(): for p in ["a(b|c)*d", "(a|aa)*b", "[^0-9]{2,3}", "\\w+@\\w+", "(a|)b"]: r = Regex(p) print("%-14r %3d NFA states" % (p, r.n_states)) for s in ["abcd", "abbcd", "aaa", "x7", "b", ""]: print(" fullmatch(%-8r) = %-5s search -> %r" % (s, r.fullmatch(s), r.search(s))) def main(argv=None): ap = argparse.ArgumentParser(description="Thompson-NFA regex engine") ap.add_argument("--check", action="store_true", help="run unit tests + differential fuzz + mutation test") ap.add_argument("--demo", action="store_true", help="print a small demo") ap.add_argument("--bench", action="store_true", help="measure re's backtracking blowup vs this engine") ap.add_argument("-p", "--pattern", help="pattern to match") ap.add_argument("-s", "--string", default="", help="subject string") ap.add_argument("--fuzz", type=int, help="ad-hoc fuzz case count") ap.add_argument("--seed", type=int, default=20261021, help="fuzz seed") args = ap.parse_args(argv) if args.check: return run_check() if args.demo: demo() return 0 if args.bench: return bench() if args.fuzz is not None: mm, skips = fuzz(args.fuzz, args.seed) print("cases=%d mismatches=%d unparseable-by-re=%d" % (args.fuzz, len(mm), skips)) for p, s, mine, theirs in mm[:10]: print(" %r on %r: engine=%r re=%r" % (p, s, mine, theirs)) return 1 if mm else 0 if args.pattern is not None: r = Regex(args.pattern) print("pattern : %r (%d NFA states)" % (args.pattern, r.n_states)) print("fullmatch : %s" % r.fullmatch(args.string)) print("match : %r" % (r.match(args.string),)) print("search : %r" % (r.search(args.string),)) return 0 ap.print_help() return 2 if __name__ == "__main__": sys.exit(main())