84 lines
3.3 KiB
Python
84 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""evidence_run.py -- reproduce every number quoted in EVIDENCE.txt, in one run.
|
|
|
|
python3 evidence_run.py (exit 0 = all sections ran clean)
|
|
|
|
Wizard: null Lane: Codeforge (hard coding and agentic reasoning)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import os
|
|
import random
|
|
import sys
|
|
import unittest
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
|
|
import datalog # noqa: E402
|
|
import test_datalog as T # noqa: E402
|
|
|
|
|
|
def main() -> int:
|
|
print("== 1. full unittest suite (the CHECK command runs this same suite)")
|
|
suite = unittest.defaultTestLoader.loadTestsFromModule(T)
|
|
stream = io.StringIO()
|
|
result = unittest.TextTestRunner(stream=stream, verbosity=1).run(suite)
|
|
print(stream.getvalue().strip())
|
|
print(f"testsRun={result.testsRun} failures={len(result.failures)} "
|
|
f"errors={len(result.errors)}")
|
|
|
|
print("\n== 2. semi-naive vs naive work on a chain transitive closure")
|
|
print(f"{'n':>3} {'|path|':>7} {'semi-naive':>11} {'naive':>10} {'speedup':>8} {'rounds':>7}")
|
|
for n in (6, 12, 16, 20):
|
|
program = T.WorkReductionTests.chain_program(n)
|
|
engine_stats, oracle_stats = {}, {}
|
|
got = datalog.evaluate(program, engine_stats)
|
|
want = T.oracle_evaluate(program, oracle_stats)
|
|
assert got == want, f"engine/oracle disagree at n={n}"
|
|
ratio = oracle_stats["bindings"] / max(engine_stats["bindings"], 1)
|
|
print(f"{n:>3} {len(got['path']):>7} {engine_stats['bindings']:>11} "
|
|
f"{oracle_stats['bindings']:>10} {ratio:>7.1f}x {engine_stats['rounds']:>7}")
|
|
|
|
print("\n== 3. fresh-seed differential run: 400 programs, engine vs oracle")
|
|
rng = random.Random(999)
|
|
programs = ([T.gen_program(rng, constrained=True) for _ in range(200)]
|
|
+ [T.gen_program(rng, constrained=False) for _ in range(200)])
|
|
counters = {"programs": 0, "unstratified": 0, "predicates": 0}
|
|
failures = T.differential_run(programs, counters)
|
|
print(f"programs={counters['programs']} unstratified={counters['unstratified']} "
|
|
f"predicate-relations compared={counters['predicates']} "
|
|
f"mismatches={len(failures)}")
|
|
for idx, text, why in failures[:3]:
|
|
print(f"MISMATCH #{idx}: {why}\n{text}")
|
|
|
|
print("\n== 4. strata assignment + refusal of a non-stratified program")
|
|
program = datalog.parse_program(
|
|
"node(a). node(b). node(c). source(a). edge(a,b).\n"
|
|
"reach(X) :- source(X).\n"
|
|
"reach(X) :- edge(Y,X), reach(Y).\n"
|
|
"unreach(X) :- node(X), not reach(X).\n"
|
|
)
|
|
print("strata:", [sorted(w) for w in datalog.stratify(program)])
|
|
rels = datalog.evaluate(program)
|
|
print("reach:", sorted(rels["reach"]), " unreach:", sorted(rels["unreach"]))
|
|
try:
|
|
datalog.stratify(datalog.parse_program(
|
|
"dom(a). p(X) :- dom(X), not q(X). q(X) :- p(X).\n"
|
|
))
|
|
print("ERROR: unstratified program was accepted")
|
|
return 1
|
|
except datalog.StratificationError as exc:
|
|
print("refused:", exc)
|
|
|
|
ok = (len(result.failures) == 0 and len(result.errors) == 0
|
|
and not failures and result.testsRun >= 40)
|
|
print("\nALL SECTIONS CLEAN" if ok else "\nSOMETHING FAILED")
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|