125 lines
4.3 KiB
Python
125 lines
4.3 KiB
Python
"""Pytest configuration for the test suite."""
|
|
import re
|
|
import pytest
|
|
|
|
# Configure pytest-asyncio mode
|
|
pytest_plugins = ["pytest_asyncio"]
|
|
|
|
# Pattern that constitutes a valid issue link in a skip reason.
|
|
# Accepts: #NNN, https?://..., or JIRA-NNN style keys.
|
|
_ISSUE_LINK_RE = re.compile(
|
|
r"(#\d+|https?://\S+|[A-Z]+-\d+)",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _has_issue_link(reason: str) -> bool:
|
|
"""Return True if *reason* contains a recognisable issue reference."""
|
|
return bool(_ISSUE_LINK_RE.search(reason or ""))
|
|
|
|
|
|
def _skip_reason(report) -> str:
|
|
"""Extract the human-readable skip reason from a pytest report."""
|
|
longrepr = getattr(report, "longrepr", None)
|
|
if longrepr is None:
|
|
return ""
|
|
if isinstance(longrepr, tuple) and len(longrepr) >= 3:
|
|
# (filename, lineno, "Skipped: <reason>")
|
|
return str(longrepr[2])
|
|
return str(longrepr)
|
|
|
|
|
|
def pytest_configure(config):
|
|
"""Configure pytest."""
|
|
config.addinivalue_line(
|
|
"markers", "integration: mark test as integration test (requires MCP servers)"
|
|
)
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"quarantine: mark test as quarantined (flaky/broken, tracked by issue)",
|
|
)
|
|
|
|
|
|
def pytest_addoption(parser):
|
|
"""Add custom command-line options."""
|
|
parser.addoption(
|
|
"--run-integration",
|
|
action="store_true",
|
|
default=False,
|
|
help="Run integration tests that require MCP servers",
|
|
)
|
|
parser.addoption(
|
|
"--no-skip-enforcement",
|
|
action="store_true",
|
|
default=False,
|
|
help="Disable poka-yoke enforcement of issue-linked skip reasons (CI escape hatch)",
|
|
)
|
|
|
|
|
|
def pytest_collection_modifyitems(config, items):
|
|
"""Modify test collection based on options."""
|
|
if not config.getoption("--run-integration"):
|
|
skip_integration = pytest.mark.skip(
|
|
reason="Integration tests require --run-integration and MCP servers running"
|
|
)
|
|
for item in items:
|
|
if "integration" in item.keywords:
|
|
item.add_marker(skip_integration)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# POKA-YOKE: Treat skipped tests as failures unless they carry an issue link.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@pytest.hookimpl(hookwrapper=True)
|
|
def pytest_runtest_makereport(item, call):
|
|
"""Intercept skipped reports and fail them if they lack an issue link.
|
|
|
|
Exceptions:
|
|
* Tests in tests/quarantine/ — explicitly quarantined, issue link required
|
|
on the quarantine marker, not the skip marker.
|
|
* Tests using environment-variable-based ``skipif`` conditions — these are
|
|
legitimate CI gates (RUN_INTEGRATION_TESTS, RUN_LIVE_TESTS, etc.) where
|
|
the *condition* is the gate, not a developer opt-out. We allow these
|
|
only when the skip reason mentions a recognised env-var pattern.
|
|
* --no-skip-enforcement flag set (emergency escape hatch).
|
|
"""
|
|
outcome = yield
|
|
report = outcome.get_result()
|
|
|
|
if not report.skipped:
|
|
return
|
|
|
|
# Escape hatch for emergency use.
|
|
if item.config.getoption("--no-skip-enforcement", default=False):
|
|
return
|
|
|
|
reason = _skip_reason(report)
|
|
|
|
# Allow quarantined tests — they are tracked by their quarantine marker.
|
|
if "quarantine" in item.keywords:
|
|
return
|
|
|
|
# Allow env-var-gated skipif conditions. These come from the
|
|
# pytest_collection_modifyitems integration gate above, or from
|
|
# explicit @pytest.mark.skipif(..., reason="... requires ENV=1 ...")
|
|
_ENV_GATE_RE = re.compile(r"(require|needs|set)\s+\w+=[^\s]+", re.IGNORECASE)
|
|
if _ENV_GATE_RE.search(reason):
|
|
return
|
|
|
|
# Allow skips added by the integration gate in this very conftest.
|
|
if "require --run-integration" in reason:
|
|
return
|
|
|
|
# Anything else needs an issue link.
|
|
if not _has_issue_link(reason):
|
|
report.outcome = "failed"
|
|
report.longrepr = (
|
|
"[POKA-YOKE] Skip without issue link is not allowed.\n"
|
|
f" Reason given: {reason!r}\n"
|
|
" Fix: add an issue reference to the skip reason, e.g.:\n"
|
|
" @pytest.mark.skip(reason='Broken until #NNN is resolved')\n"
|
|
" Or quarantine the test: move it to tests/quarantine/ and\n"
|
|
" file an issue — see docs/QUARANTINE_PROCESS.md"
|
|
)
|