72 lines
2.0 KiB
Python
72 lines
2.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Find duplicate files by content hash.
|
|
Run: python3 duplicate_finder.py <directory> [max_depth=2]
|
|
"""
|
|
import hashlib
|
|
import os
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
|
|
def hash_file(path, algo="sha256", blocksize=65536):
|
|
h = hashlib.new(algo)
|
|
with open(path, "rb") as f:
|
|
for block in iter(lambda: f.read(blocksize), b""):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def find_duplicates(root_dir, max_depth=2):
|
|
root = Path(root_dir)
|
|
hashes = defaultdict(list)
|
|
|
|
for p in root.rglob("*"):
|
|
if not p.is_file():
|
|
continue
|
|
if p.name.startswith(".") or p.name.startswith("__pycache__"):
|
|
continue
|
|
|
|
depth = len(p.relative_to(root).parts) - 1
|
|
if depth > max_depth:
|
|
continue
|
|
|
|
try:
|
|
h = hash_file(p)
|
|
hashes[h].append(str(p))
|
|
except (PermissionError, OSError) as e:
|
|
print(f"SKIP: {p}: {e}", file=sys.stderr)
|
|
|
|
return {h: paths for h, paths in hashes.items() if len(paths) > 1}
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 2:
|
|
print("Usage: python3 duplicate_finder.py <directory> [max_depth=2]", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
root_dir = sys.argv[1]
|
|
max_depth = int(sys.argv[2]) if len(sys.argv) > 2 else 2
|
|
|
|
if not os.path.isdir(root_dir):
|
|
print(f"Error: {root_dir} is not a directory", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
dups = find_duplicates(root_dir, max_depth)
|
|
|
|
if dups:
|
|
print(f"Found {len(dups)} duplicate group(s):")
|
|
for h, paths in dups.items():
|
|
print(f"\n[hash: {h[:12]}...]")
|
|
for p in sorted(paths):
|
|
try:
|
|
size = os.path.getsize(p)
|
|
print(f" {p} ({size} bytes)")
|
|
except OSError:
|
|
print(f" {p}")
|
|
else:
|
|
print(f"No duplicates found in {root_dir} (depth <= {max_depth})")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |