Alexander Whitestone
10eb29f57b
Post-deployment UI verification using vision: - Screenshot-based page rendering checks - Error detection (404, 500, broken layouts) - Expected content verification - Batch URL checking for multi-endpoint deployments Relates to #1485
147 lines
4.5 KiB
Python
147 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Deployment Visual Verification
|
|
==============================
|
|
|
|
Post-deployment step that uses vision to verify UI is rendered correctly.
|
|
Takes screenshots of deployed endpoints and checks for:
|
|
- Page rendering errors
|
|
- Missing assets
|
|
- Layout breaks
|
|
- Error messages visible
|
|
- Expected content present
|
|
|
|
Usage:
|
|
python scripts/deploy_verify.py check https://my-app.com
|
|
python scripts/deploy_verify.py check https://my-app.com --expect "Welcome"
|
|
python scripts/deploy_verify.py batch urls.txt
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
|
|
@dataclass
|
|
class DeployCheck:
|
|
"""A single deployment verification check."""
|
|
url: str
|
|
status: str # passed, failed, warning
|
|
issues: list = field(default_factory=list)
|
|
screenshot_path: Optional[str] = None
|
|
expected_content: str = ""
|
|
timestamp: str = ""
|
|
|
|
def summary(self) -> str:
|
|
emoji = {"passed": "✅", "failed": "❌", "warning": "⚠️"}.get(self.status, "❓")
|
|
lines = [
|
|
f"{emoji} {self.url}",
|
|
f" Checked: {self.timestamp or 'pending'}",
|
|
]
|
|
if self.expected_content:
|
|
lines.append(f" Expected: '{self.expected_content}'")
|
|
if self.issues:
|
|
lines.append(" Issues:")
|
|
for i in self.issues:
|
|
lines.append(f" - {i}")
|
|
else:
|
|
lines.append(" No issues detected")
|
|
return "\n".join(lines)
|
|
|
|
|
|
class DeployVerifier:
|
|
"""Verifies deployed UI renders correctly using screenshots."""
|
|
|
|
def build_check_prompt(self, url: str, expected: str = "") -> dict:
|
|
"""Build verification prompt for a deployed URL."""
|
|
expect_clause = ""
|
|
if expected:
|
|
expect_clause = f"\n- Verify the text \"{expected}\" is visible on the page"
|
|
|
|
prompt = f"""Take a screenshot of {url} and verify the deployment is healthy.
|
|
|
|
Check for:
|
|
- Page loads without errors (no 404, 500, connection refused)
|
|
- No visible error messages or stack traces
|
|
- Layout is not broken (elements properly aligned, no overlapping)
|
|
- Images and assets load correctly (no broken image icons)
|
|
- Navigation elements are present and clickable{expect_clause}
|
|
- No "under construction" or placeholder content
|
|
- Responsive design elements render properly
|
|
|
|
Return as JSON:
|
|
```json
|
|
{{
|
|
"status": "passed|failed|warning",
|
|
"issues": ["list of issues found"],
|
|
"confidence": 0.9,
|
|
"page_title": "detected page title",
|
|
"visible_text_sample": "first 100 chars of visible text"
|
|
}}
|
|
```
|
|
"""
|
|
return {
|
|
"url": url,
|
|
"prompt": prompt,
|
|
"screenshot_needed": True,
|
|
"instruction": f"browser_navigate to {url}, take screenshot with browser_vision, analyze with prompt"
|
|
}
|
|
|
|
def verify_deployment(self, url: str, expected: str = "", screenshot_path: str = "") -> DeployCheck:
|
|
"""Create a deployment verification check."""
|
|
check = DeployCheck(
|
|
url=url,
|
|
status="pending",
|
|
expected_content=expected,
|
|
timestamp=datetime.now().isoformat(),
|
|
screenshot_path=screenshot_path or f"/tmp/deploy_verify_{url.replace('://', '_').replace('/', '_')}.png"
|
|
)
|
|
return check
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 2:
|
|
print("Usage: deploy_verify.py <check|batch> [args...]")
|
|
return 1
|
|
|
|
verifier = DeployVerifier()
|
|
cmd = sys.argv[1]
|
|
|
|
if cmd == "check":
|
|
if len(sys.argv) < 3:
|
|
print("Usage: deploy_verify.py check <url> [--expect 'text']")
|
|
return 1
|
|
url = sys.argv[2]
|
|
expected = ""
|
|
if "--expect" in sys.argv:
|
|
idx = sys.argv.index("--expect")
|
|
if idx + 1 < len(sys.argv):
|
|
expected = sys.argv[idx + 1]
|
|
|
|
result = verifier.build_check_prompt(url, expected)
|
|
print(json.dumps(result, indent=2))
|
|
|
|
elif cmd == "batch":
|
|
if len(sys.argv) < 3:
|
|
print("Usage: deploy_verify.py batch <urls_file>")
|
|
return 1
|
|
urls_file = Path(sys.argv[2])
|
|
if not urls_file.exists():
|
|
print(f"File not found: {urls_file}")
|
|
return 1
|
|
|
|
urls = [line.strip() for line in urls_file.read_text().splitlines() if line.strip() and not line.startswith("#")]
|
|
for url in urls:
|
|
print(f"\n--- {url} ---")
|
|
result = verifier.build_check_prompt(url)
|
|
print(json.dumps(result, indent=2))
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|