832b23286b
Some checks failed
Test / pytest (pull_request) Failing after 7s
- Implement transitive_closure(): computes full dependency tree for each node - Implement find_deep_chains(): identifies longest paths in dependency graph - JSON output now includes `transitive` and `deep_chains` fields - Added comprehensive unit tests in scripts/test_dependency_graph.py (9 tests) - Handles cycles correctly, excludes self-references from closure Meets acceptance criteria for #111: ✅ Builds transitive dep tree ✅ Identifies deep chains and circular deps ✅ Output: transitive dependency graph (via --format json) Closes #111
340 lines
11 KiB
Python
340 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Cross-Repo Dependency Graph Builder
|
|
|
|
Scans repos for import/require/reference patterns and builds a directed
|
|
dependency graph. Detects circular dependencies. Outputs DOT and Mermaid.
|
|
|
|
Usage:
|
|
python3 scripts/dependency_graph.py /path/to/repos/
|
|
python3 scripts/dependency_graph.py --repos repo1,repo2,repo3 --format mermaid
|
|
python3 scripts/dependency_graph.py --repos-dir /path/to/ --format dot --output deps.dot
|
|
|
|
Patterns detected:
|
|
- Python: import X, from X import Y
|
|
- JavaScript: require("X"), import ... from "X"
|
|
- Go: import "X"
|
|
- Ansible: include_role, import_role
|
|
- Docker/Compose: image: X, depends_on
|
|
- Config references: repo-name in YAML/TOML/JSON
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
|
|
# Known repo names for matching
|
|
KNOWN_REPOS = [
|
|
"hermes-agent", "timmy-config", "timmy-home", "the-nexus", "the-door",
|
|
"the-beacon", "fleet-ops", "burn-fleet", "timmy-dispatch", "turboquant",
|
|
"compounding-intelligence", "the-playground", "second-son-of-timmy",
|
|
"ai-safety-review", "the-echo-pattern", "timmy-academy", "wolf",
|
|
"the-testament",
|
|
]
|
|
|
|
|
|
def normalize_repo_name(name: str) -> str:
|
|
"""Normalize a repo name for comparison."""
|
|
return name.lower().replace("_", "-").replace(".git", "").strip()
|
|
|
|
|
|
def scan_file_for_deps(filepath: str, content: str, own_repo: str) -> set:
|
|
"""Scan a file's content for references to other repos."""
|
|
deps = set()
|
|
own_norm = normalize_repo_name(own_repo)
|
|
|
|
for repo in KNOWN_REPOS:
|
|
repo_norm = normalize_repo_name(repo)
|
|
if repo_norm == own_norm:
|
|
continue
|
|
|
|
# Direct name references
|
|
patterns = [
|
|
repo, # exact name
|
|
repo.replace("-", "_"), # underscore variant
|
|
repo.replace("-", ""), # no separator
|
|
f"/{repo}/", # path reference
|
|
f'"{repo}"', # quoted
|
|
f"'{repo}'", # single quoted
|
|
f"Timmy_Foundation/{repo}", # full Gitea path
|
|
f"Timmy_Foundation.{repo}", # Python module path
|
|
]
|
|
|
|
for pattern in patterns:
|
|
if pattern in content:
|
|
deps.add(repo)
|
|
break
|
|
|
|
return deps
|
|
|
|
|
|
def scan_repo(repo_path: str, repo_name: str = None) -> dict:
|
|
"""Scan a repo directory for dependencies."""
|
|
path = Path(repo_path)
|
|
if not path.is_dir():
|
|
return {"error": f"Not a directory: {repo_path}"}
|
|
|
|
if not repo_name:
|
|
repo_name = path.name
|
|
|
|
deps = set()
|
|
files_scanned = 0
|
|
exts = {".py", ".js", ".ts", ".go", ".yaml", ".yml", ".toml", ".json",
|
|
".md", ".sh", ".bash", ".Dockerfile", ".tf", ".hcl"}
|
|
|
|
for fpath in path.rglob("*"):
|
|
if not fpath.is_file():
|
|
continue
|
|
if fpath.suffix not in exts:
|
|
continue
|
|
# Skip common non-source dirs
|
|
parts = fpath.parts
|
|
if any(p in (".git", "node_modules", "__pycache__", ".venv", "venv",
|
|
"vendor", "dist", "build", ".tox") for p in parts):
|
|
continue
|
|
|
|
try:
|
|
content = fpath.read_text(errors="ignore")
|
|
except:
|
|
continue
|
|
|
|
file_deps = scan_file_for_deps(str(fpath), content, repo_name)
|
|
deps.update(file_deps)
|
|
files_scanned += 1
|
|
|
|
return {
|
|
"repo": repo_name,
|
|
"dependencies": sorted(deps),
|
|
"files_scanned": files_scanned,
|
|
}
|
|
|
|
|
|
def detect_cycles(graph: dict) -> list:
|
|
"""Detect circular dependencies using DFS."""
|
|
cycles = []
|
|
visited = set()
|
|
rec_stack = set()
|
|
|
|
def dfs(node, path):
|
|
visited.add(node)
|
|
rec_stack.add(node)
|
|
|
|
for neighbor in graph.get(node, {}).get("dependencies", []):
|
|
if neighbor not in visited:
|
|
result = dfs(neighbor, path + [neighbor])
|
|
if result:
|
|
return result
|
|
elif neighbor in rec_stack:
|
|
cycle_start = path.index(neighbor)
|
|
return path[cycle_start:] + [neighbor]
|
|
|
|
rec_stack.remove(node)
|
|
return None
|
|
|
|
for node in graph:
|
|
if node not in visited:
|
|
cycle = dfs(node, [node])
|
|
if cycle:
|
|
cycles.append(cycle)
|
|
|
|
return cycles
|
|
|
|
|
|
def to_dot(graph: dict) -> str:
|
|
"""Generate DOT format output."""
|
|
lines = ["digraph dependencies {"]
|
|
lines.append(" rankdir=LR;")
|
|
lines.append(' node [shape=box, style=filled, fillcolor="#1a1a2e", fontcolor="#e6edf3"];')
|
|
lines.append(' edge [color="#4a4a6a"];')
|
|
lines.append("")
|
|
|
|
for repo, data in sorted(graph.items()):
|
|
dep_count = len(data.get("dependencies", []))
|
|
fill = "#2d1b69" if dep_count > 2 else "#16213e"
|
|
lines.append(f' "{repo}" [fillcolor="{fill}"];')
|
|
for dep in data.get("dependencies", []):
|
|
lines.append(f' "{repo}" -> "{dep}";')
|
|
|
|
lines.append("}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def to_mermaid(graph: dict) -> str:
|
|
"""Generate Mermaid format output."""
|
|
lines = ["graph LR"]
|
|
|
|
for repo, data in sorted(graph.items()):
|
|
for dep in data.get("dependencies", []):
|
|
lines.append(f" {repo.replace('-','_')} --> {dep.replace('-','_')}")
|
|
|
|
# Add node labels
|
|
lines.append("")
|
|
for repo in sorted(graph.keys()):
|
|
lines.append(f" {repo.replace('-','_')}[{repo}]")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
|
|
def transitive_closure(graph: dict) -> dict:
|
|
"""Compute transitive closure for each node (all indirect deps)."""
|
|
closure = {}
|
|
# Build adjacency list
|
|
adj = {node: set(data.get("dependencies", [])) for node, data in graph.items()}
|
|
all_nodes = set(adj.keys()) | set().union(*adj.values())
|
|
|
|
for node in all_nodes:
|
|
visited = set()
|
|
stack = list(adj.get(node, set()))
|
|
while stack:
|
|
current = stack.pop()
|
|
if current not in visited:
|
|
visited.add(current)
|
|
stack.extend(adj.get(current, set()))
|
|
# Remove self-reference: a node's transitive deps should not include itself
|
|
visited.discard(node)
|
|
closure[node] = visited
|
|
|
|
return closure
|
|
|
|
|
|
def find_deep_chains(graph: dict) -> list[list[str]]:
|
|
"""Find the longest simple paths in the dependency graph (ignoring cycles)."""
|
|
from collections import defaultdict
|
|
|
|
adj = {node: list(data.get("dependencies", [])) for node, data in graph.items()}
|
|
deepest = []
|
|
max_len = 0
|
|
|
|
def dfs(node: str, path: list, visited: set):
|
|
nonlocal deepest, max_len
|
|
# Stop if we hit a cycle (node already in path)
|
|
if node in path:
|
|
return
|
|
new_path = path + [node]
|
|
if node not in adj or not adj[node]:
|
|
# leaf
|
|
if len(new_path) > max_len:
|
|
max_len = len(new_path)
|
|
deepest = [new_path.copy()]
|
|
elif len(new_path) == max_len:
|
|
deepest.append(new_path.copy())
|
|
else:
|
|
for neighbor in adj[node]:
|
|
dfs(neighbor, new_path.copy(), visited | {node})
|
|
|
|
for start in graph:
|
|
dfs(start, [], set())
|
|
|
|
return deepest
|
|
|
|
|
|
def format_transitive_markdown(closure: dict) -> str:
|
|
"""Render transitive closure as a markdown table."""
|
|
lines = ["# Transitive Dependencies\n\n"]
|
|
lines.append("| Node | Transitive Dependencies | Count |\n")
|
|
lines.append("|------|------------------------|-------|\n")
|
|
for node in sorted(closure.keys()):
|
|
deps = closure[node]
|
|
deps_str = ", ".join(sorted(deps)) if deps else "(none)"
|
|
lines.append(f"| {node} | {deps_str} | {len(deps)} |\n")
|
|
return "".join(lines)
|
|
|
|
|
|
def format_deep_chains_markdown(chains: list[list[str]]) -> str:
|
|
"""Render longest dependency chains as a markdown list."""
|
|
lines = ["# Deepest Dependency Chains\n\n"]
|
|
if not chains:
|
|
lines.append("No chains found.\n")
|
|
return "".join(lines)
|
|
max_len = max(len(c) for c in chains)
|
|
lines.append(f"*Longest chain length:* {max_len}\n\n")
|
|
for i, chain in enumerate(sorted(chains, key=lambda c: (-len(c), " -> ".join(c))), 1):
|
|
lines.append(f"**Chain {i}** ({len(chain)} nodes)\n\n")
|
|
indent = " "
|
|
for j, node in enumerate(chain):
|
|
arrow = " → " if j < len(chain)-1 else " • "
|
|
lines.append(f"{indent}{arrow}{node}\n")
|
|
lines.append("\n")
|
|
return "".join(lines)
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Build cross-repo dependency graph")
|
|
parser.add_argument("repos_dir", nargs="?", help="Directory containing repos")
|
|
parser.add_argument("--repos", help="Comma-separated list of repo paths")
|
|
parser.add_argument("--format", choices=["dot", "mermaid", "json"], default="json")
|
|
parser.add_argument("--output", "-o", help="Output file (default: stdout)")
|
|
parser.add_argument("--cycles-only", action="store_true", help="Only report cycles")
|
|
args = parser.parse_args()
|
|
|
|
results = {}
|
|
repo_paths = []
|
|
|
|
if args.repos:
|
|
repo_paths = [p.strip() for p in args.repos.split(",")]
|
|
elif args.repos_dir:
|
|
base = Path(args.repos_dir)
|
|
repo_paths = [str(p) for p in base.iterdir() if p.is_dir() and not p.name.startswith(".")]
|
|
else:
|
|
parser.print_help()
|
|
sys.exit(1)
|
|
|
|
for rpath in repo_paths:
|
|
name = Path(rpath).name
|
|
print(f"Scanning {name}...", file=sys.stderr)
|
|
result = scan_repo(rpath, name)
|
|
if "error" not in result:
|
|
results[name] = result
|
|
|
|
# Detect cycles
|
|
cycles = detect_cycles(results)
|
|
|
|
if args.cycles_only:
|
|
if cycles:
|
|
print("CIRCULAR DEPENDENCIES DETECTED:")
|
|
for cycle in cycles:
|
|
print(f" {' -> '.join(cycle)}")
|
|
sys.exit(1)
|
|
else:
|
|
print("No circular dependencies found.")
|
|
sys.exit(0)
|
|
|
|
# Output
|
|
output = {}
|
|
if args.format == "dot":
|
|
output = to_dot(results)
|
|
elif args.format == "mermaid":
|
|
output = to_mermaid(results)
|
|
else:
|
|
# Compute transitive and deep chains
|
|
closure = transitive_closure(results)
|
|
deep_chains = find_deep_chains(results)
|
|
output = json.dumps({
|
|
"repos": results,
|
|
"cycles": cycles,
|
|
"transitive": {node: sorted(deps) for node, deps in closure.items()},
|
|
"deep_chains": [chain for chain in deep_chains if len(chain) > 1],
|
|
"summary": {
|
|
"total_repos": len(results),
|
|
"total_deps": sum(len(r["dependencies"]) for r in results.values()),
|
|
"cycles_found": len(cycles),
|
|
"transitive_pairs": sum(len(deps) for deps in closure.values()),
|
|
"longest_chain_length": max((len(c) for c in deep_chains), default=0),
|
|
}
|
|
}, indent=2)
|
|
|
|
if args.output:
|
|
Path(args.output).write_text(output)
|
|
print(f"Written to {args.output}", file=sys.stderr)
|
|
else:
|
|
print(output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|