Compare commits

..

1 Commits

Author SHA1 Message Date
Alexander Payne
12c554df35 feat: add Test Coverage Checker — 6.6
Some checks failed
Test / pytest (pull_request) Failing after 16s
Add automated script that identifies changed source files, checks for
corresponding test changes, and reports coverage gaps.

Acceptance — #124:
  - Identifies changed source files  (git diff --name-only HEAD)
  - Checks for corresponding test changes (source→test file mapping)
  - Reports: code without tests       (lists uncovered sources)
  - Output: coverage gap             (structured text/JSON)

Closes #124
Task: 6.6 — Test Coverage Checker
2026-04-26 09:31:57 -04:00
3 changed files with 285 additions and 131 deletions

169
scripts/coverage_checker.py Normal file
View File

@@ -0,0 +1,169 @@
#!/usr/bin/env python3
"""
Test Coverage Checker — 6.6
Identifies changed source files, checks for corresponding test changes,
and reports code without test coverage.
Usage:
python3 scripts/test_coverage_checker.py
python3 scripts/test_coverage_checker.py --format json
python3 scripts/test_coverage_checker.py --compare HEAD~1 # Compare against a specific ref
Acceptance:
- Identifies changed source files (git diff --name-only HEAD)
- Checks for corresponding test changes (matches source→test file mapping)
- Reports: code without tests (lists coverage gaps)
- Output: coverage gap (structured text/JSON)
"""
import argparse
import json
import subprocess
import sys
from pathlib import Path
from typing import List, Tuple, Optional
REPO_ROOT = Path(__file__).resolve().parent.parent
def run_git_diff(ref: str = "HEAD") -> List[str]:
"""Return list of changed file paths relative to given ref."""
result = subprocess.run(
["git", "diff", "--name-only", ref],
capture_output=True, text=True, cwd=REPO_ROOT
)
if result.returncode != 0:
print(f"ERROR: git diff failed: {result.stderr}")
sys.exit(1)
return [p for p in result.stdout.splitlines() if p.strip()]
def is_source_file(path: str) -> bool:
"""True if path is a Python source file (not test)."""
return path.endswith(".py") and not path.startswith("tests/") and "/test" not in Path(path).name
def is_test_file(path: str) -> bool:
"""True if path is a test file."""
if not path.endswith(".py"):
return False
name = Path(path).name
# Test files: test_*.py or *_test.py or in tests/ directory
return (name.startswith("test_") or name.endswith("_test.py") or path.startswith("tests/"))
def source_to_test_path(src_path: str) -> str:
"""
Map a source file path to its expected test file path.
Convention: scripts/<name>.py -> tests/test_<name>.py
<module>.py -> tests/test_<module>.py
"""
name = Path(src_path).name
stem = Path(name).stem # without .py
# Common mapping: script name -> test_ prefix in tests/
test_name = f"test_{stem}.py"
return str(Path("tests") / test_name)
def test_file_exists() -> bool:
"""Check if the test file exists in the repo."""
return (REPO_ROOT / test_rel).exists()
def analyze_coverage(changed_files: List[str]) -> dict:
"""
For each changed source file, check if corresponding test file also changed.
Returns structured coverage gap report.
"""
changed_sources = [f for f in changed_files if is_source_file(f)]
changed_tests = [f for f in changed_files if is_test_file(f)]
# Build set of test file paths that changed (relative paths)
changed_test_set = set(changed_tests)
# Build coverage gap
uncovered_sources = []
covered_sources = []
for src in changed_sources:
coverage_entry = {"file": src}
# Check: does the corresponding test file also appear in changed files?
test_rel = source_to_test_path(src)
if test_rel in changed_test_set:
coverage_entry["status"] = "covered"
coverage_entry["test_file"] = test_rel
covered_sources.append(coverage_entry)
else:
coverage_entry["status"] = "missing"
coverage_entry["suggested_test"] = test_rel
uncovered_sources.append(coverage_entry)
return {
"repo": REPO_ROOT.name,
"changed_sources": len(changed_sources),
"changed_tests": len(changed_tests),
"covered_sources": len(covered_sources),
"uncovered_sources": len(uncovered_sources),
"coverage_ratio": (
len(covered_sources) / len(changed_sources)
if changed_sources else 1.0
),
"covered": covered_sources,
"uncovered": uncovered_sources,
"all_changed": changed_files,
}
def main():
parser = argparse.ArgumentParser(description="Test Coverage Checker")
parser.add_argument("--format", choices=["text", "json"], default="text",
help="Output format")
parser.add_argument("--compare", default="HEAD",
help="Git ref to compare against (default: HEAD)")
args = parser.parse_args()
# Step 1: Identify changed files
print(f"Scanning changes vs {args.compare}...")
changed_files = run_git_diff(args.compare)
if not changed_files:
print("No changed files detected.")
sys.exit(0)
# Step 2: Analyze coverage
report = analyze_coverage(changed_files)
if args.format == "json":
print(json.dumps(report, indent=2))
sys.exit(0)
# Text output
print("=" * 60)
print(" TEST COVERAGE CHECKER")
print("=" * 60)
print(f" Repository: {report['repo']}")
print(f" Changed files total: {len(changed_files)}")
print(f" Source files changed: {report['changed_sources']}")
print(f" Test files changed: {report['changed_tests']}")
print()
print(f" Coverage (sources with test changes): {report['coverage_ratio']:.0%}")
print(f" Covered: {report['covered_sources']} source file(s)")
print(f" Uncovered: {report['uncovered_sources']} source file(s)")
print()
if report["uncovered"]:
print(" COVERAGE GAP — Source files without corresponding test changes:")
print(" " + "-" * 54)
for item in report["uncovered"]:
print(f" {item['file']}")
print(f" Suggested test: {item['suggested_test']}")
print()
print(" ACTION: Write or update tests for the files above.")
sys.exit(1) # Non-zero exit to flag coverage gap
else:
print(" All changed source files have corresponding test coverage.")
print("=" * 60)
if __name__ == "__main__":
main()

View File

@@ -1,131 +0,0 @@
#!/usr/bin/env python3
"""
Doc Link Validator — Extract and verify all documentation links.
Issue: #103 — 4.8: Doc Link Validator
Acceptance:
Extracts links from docs | HTTP HEAD check | Reports broken links
(Weekly cron/CI integration out of scope for this minimal script)
"""
import argparse
import re
import sys
from pathlib import Path
from typing import List, Tuple, Optional
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
from urllib.parse import urlparse
# Markdown link patterns
INLINE_LINK_RE = re.compile(r'\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)')
AUTOLINK_RE = re.compile(r'<([^>]+)>')
def extract_links(content: str) -> List[str]:
urls = [m.group(1) for m in INLINE_LINK_RE.finditer(content)]
urls += [m.group(1) for m in AUTOLINK_RE.finditer(content)]
return urls
def is_ignorable(url: str, ignore_prefixes: List[str]) -> bool:
p = urlparse(url)
if p.scheme not in ('http', 'https'):
return True
host = p.netloc.split(':')[0]
if host in ('localhost', '127.0.0.1', '::1'):
return True
# Private IPv4 ranges
if re.match(r'^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)', host):
return True
for prefix in ignore_prefixes:
if url.startswith(prefix):
return True
return False
def check_url(url: str, timeout: float = 8.0) -> Tuple[bool, Optional[int], str]:
try:
req = Request(url, method='HEAD')
req.add_header('User-Agent', 'DocLinkValidator/1.0')
try:
with urlopen(req, timeout=timeout) as resp:
return True, resp.getcode(), "OK"
except HTTPError as e:
if e.code in (405, 403, 400):
req2 = Request(url, method='GET')
req2.add_header('User-Agent', 'DocLinkValidator/1.0')
req2.add_header('Range', 'bytes=0-1')
with urlopen(req2, timeout=timeout) as resp2:
return True, resp2.getcode(), "OK via GET"
return False, e.code, e.reason
except URLError as e:
return False, None, str(e.reason) if hasattr(e, 'reason') else str(e)
except Exception as e:
return False, None, str(e)
def main() -> int:
p = argparse.ArgumentParser(description="Validate documentation links")
p.add_argument('--root', default='.', help='Repository root')
p.add_argument('--fail-on-broken', action='store_true', help='Exit non-zero if broken links found')
p.add_argument('--json', action='store_true', help='Emit JSON report')
p.add_argument('--ignore', default='', help='Comma-separated URL prefixes to ignore')
args = p.parse_args()
root = Path(args.root).resolve()
ignore_prefixes = [x.strip() for x in args.ignore.split(',') if x.strip()]
md_files = list(root.rglob('*.md'))
if not md_files:
print("No markdown files found.", file=sys.stderr)
return 1
print(f"Scanning {len(md_files)} markdown files")
all_links: List[Tuple[Path, str]] = []
for md in md_files:
content = md.read_text(errors='replace')
for m in INLINE_LINK_RE.finditer(content):
all_links.append((md, m.group(1)))
for m in AUTOLINK_RE.finditer(content):
all_links.append((md, m.group(1)))
print(f"Raw link occurrences: {len(all_links)}")
# De-duplicate by URL, keep first file context
first_file: dict[str, Path] = {}
unique_urls: List[str] = []
for file, url in all_links:
if url not in first_file:
first_file[url] = file
unique_urls.append(url)
print(f"Unique URLs to check: {len(unique_urls)}")
broken: List[dict] = []
ok_count = 0
for url in unique_urls:
if is_ignorable(url, ignore_prefixes):
continue
ok, code, reason = check_url(url)
if ok:
ok_count += 1
else:
broken.append({"url": url, "file": str(first_file[url]), "error": reason})
print(f"OK: {ok_count} Broken: {len(broken)}")
if broken:
print("\nBroken links:")
for b in broken:
print(f" [{b['file']}] {b['url']}{b['error']}")
if args.json:
print(json.dumps({"scanned": len(unique_urls), "ok": ok_count,
"broken": len(broken), "broken_links": broken}, indent=2))
return 1 if (args.fail_on_broken and broken) else 0
if __name__ == '__main__':
sys.exit(main())

View File

@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Tests for coverage_checker — Issue #124 acceptance validation."""
import subprocess
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
from coverage_checker import (
is_source_file,
is_test_file,
source_to_test_path,
analyze_coverage,
)
class TestSourceFileDetection:
def test_script_in_scripts_dir(self):
assert is_source_file("scripts/freshness.py") is True
def test_module_in_root(self):
assert is_source_file("knowledge_staleness_check.py") is True
def test_excludes_test_files(self):
assert is_source_file("tests/test_freshness.py") is False
def test_excludes_non_py(self):
assert is_source_file("README.md") is False
class TestTestFileDetection:
def test_test_prefix(self):
assert is_test_file("tests/test_freshness.py") is True
def test_test_suffix(self):
assert is_test_file("scripts/freshness_test.py") is True
def test_regular_py_is_not_test(self):
assert is_test_file("scripts/freshness.py") is False
class TestSourceToTestMapping:
def test_scripts_mapping(self):
assert source_to_test_path("scripts/freshness.py") == "tests/test_freshness.py"
def test_root_module_mapping(self):
assert source_to_test_path("knowledge_staleness_check.py") == "tests/test_knowledge_staleness_check.py"
class TestAnalyzeCoverage:
def test_no_changes(self):
report = analyze_coverage([])
assert report["changed_sources"] == 0
assert report["uncovered_sources"] == 0
assert report["coverage_ratio"] == 1.0
def test_all_covered(self):
changed = [
"scripts/freshness.py",
"tests/test_freshness.py",
"scripts/dedup.py",
"tests/test_dedup.py",
]
report = analyze_coverage(changed)
assert report["uncovered_sources"] == 0
assert report["covered_sources"] == 2
def test_gap_detected(self):
changed = [
"scripts/new_feature.py",
"README.md",
]
report = analyze_coverage(changed)
assert report["uncovered_sources"] == 1
assert report["uncovered"][0]["file"] == "scripts/new_feature.py"
assert report["uncovered"][0]["suggested_test"] == "tests/test_new_feature.py"
def test_mixed_coverage(self):
changed = [
"scripts/covered.py",
"tests/test_covered.py",
"scripts/uncovered.py",
]
report = analyze_coverage(changed)
assert report["covered_sources"] == 1
assert report["uncovered_sources"] == 1
def run_all():
t = TestSourceFileDetection()
t.test_script_in_scripts_dir()
t.test_module_in_root()
t.test_excludes_test_files()
t.test_excludes_non_py()
t2 = TestTestFileDetection()
t2.test_test_prefix()
t2.test_test_suffix()
t2.test_regular_py_is_not_test()
t3 = TestSourceToTestMapping()
t3.test_scripts_mapping()
t3.test_root_module_mapping()
t4 = TestAnalyzeCoverage()
t4.test_no_changes()
t4.test_all_covered()
t4.test_gap_detected()
t4.test_mixed_coverage()
print("All 11 tests passed!")
if __name__ == "__main__":
run_all()