#!/usr/bin/env python3 """Find duplicate files by content hash. Run: python3 duplicate_finder.py [max_depth=2] """ import hashlib import os import sys from collections import defaultdict from pathlib import Path def hash_file(path, algo="sha256", blocksize=65536): h = hashlib.new(algo) with open(path, "rb") as f: for block in iter(lambda: f.read(blocksize), b""): h.update(block) return h.hexdigest() def find_duplicates(root_dir, max_depth=2): root = Path(root_dir) hashes = defaultdict(list) for p in root.rglob("*"): if not p.is_file(): continue if p.name.startswith(".") or p.name.startswith("__pycache__"): continue depth = len(p.relative_to(root).parts) - 1 if depth > max_depth: continue try: h = hash_file(p) hashes[h].append(str(p)) except (PermissionError, OSError) as e: print(f"SKIP: {p}: {e}", file=sys.stderr) return {h: paths for h, paths in hashes.items() if len(paths) > 1} def main(): if len(sys.argv) < 2: print("Usage: python3 duplicate_finder.py [max_depth=2]", file=sys.stderr) sys.exit(1) root_dir = sys.argv[1] max_depth = int(sys.argv[2]) if len(sys.argv) > 2 else 2 if not os.path.isdir(root_dir): print(f"Error: {root_dir} is not a directory", file=sys.stderr) sys.exit(1) dups = find_duplicates(root_dir, max_depth) if dups: print(f"Found {len(dups)} duplicate group(s):") for h, paths in dups.items(): print(f"\n[hash: {h[:12]}...]") for p in sorted(paths): try: size = os.path.getsize(p) print(f" {p} ({size} bytes)") except OSError: print(f" {p}") else: print(f"No duplicates found in {root_dir} (depth <= {max_depth})") if __name__ == "__main__": main()