318 lines
10 KiB
Python
318 lines
10 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""
|
||
|
|
dedup.py — Knowledge deduplication: content hash + semantic similarity.
|
||
|
|
|
||
|
|
Deduplicates harvested knowledge entries to avoid training on duplicates.
|
||
|
|
Uses content hashing for exact matches and token overlap for near-duplicates.
|
||
|
|
|
||
|
|
Usage:
|
||
|
|
python3 dedup.py --input knowledge/index.json --output knowledge/index_deduped.json
|
||
|
|
python3 dedup.py --input knowledge/index.json --dry-run
|
||
|
|
python3 dedup.py --test # Run built-in dedup test
|
||
|
|
"""
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import List, Dict, Optional, Tuple
|
||
|
|
|
||
|
|
|
||
|
|
def normalize_text(text: str) -> str:
|
||
|
|
"""Normalize text for hashing: lowercase, collapse whitespace, strip."""
|
||
|
|
text = text.lower().strip()
|
||
|
|
text = re.sub(r'\s+', ' ', text)
|
||
|
|
return text
|
||
|
|
|
||
|
|
|
||
|
|
def content_hash(text: str) -> str:
|
||
|
|
"""SHA256 hash of normalized text for exact dedup."""
|
||
|
|
normalized = normalize_text(text)
|
||
|
|
return hashlib.sha256(normalized.encode('utf-8')).hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def tokenize(text: str) -> set:
|
||
|
|
"""Simple tokenizer: lowercase words, 3+ chars."""
|
||
|
|
words = re.findall(r'[a-z0-9_]{3,}', text.lower())
|
||
|
|
return set(words)
|
||
|
|
|
||
|
|
|
||
|
|
def token_similarity(a: str, b: str) -> float:
|
||
|
|
"""Token-based Jaccard similarity (0.0-1.0).
|
||
|
|
|
||
|
|
Fast local alternative to embedding similarity.
|
||
|
|
Good enough for near-duplicate detection.
|
||
|
|
"""
|
||
|
|
tokens_a = tokenize(a)
|
||
|
|
tokens_b = tokenize(b)
|
||
|
|
if not tokens_a or not tokens_b:
|
||
|
|
return 0.0
|
||
|
|
intersection = tokens_a & tokens_b
|
||
|
|
union = tokens_a | tokens_b
|
||
|
|
return len(intersection) / len(union)
|
||
|
|
|
||
|
|
|
||
|
|
def quality_score(fact: dict) -> float:
|
||
|
|
"""Compute quality score for merge ranking.
|
||
|
|
|
||
|
|
Higher is better. Factors:
|
||
|
|
- confidence (0-1)
|
||
|
|
- source_count (more confirmations = better)
|
||
|
|
- has tags (richer metadata)
|
||
|
|
"""
|
||
|
|
confidence = fact.get('confidence', 0.5)
|
||
|
|
source_count = fact.get('source_count', 1)
|
||
|
|
has_tags = 1.0 if fact.get('tags') else 0.0
|
||
|
|
has_related = 1.0 if fact.get('related') else 0.0
|
||
|
|
|
||
|
|
# Weighted composite
|
||
|
|
score = (
|
||
|
|
confidence * 0.5 +
|
||
|
|
min(source_count / 10, 1.0) * 0.3 +
|
||
|
|
has_tags * 0.1 +
|
||
|
|
has_related * 0.1
|
||
|
|
)
|
||
|
|
return round(score, 4)
|
||
|
|
|
||
|
|
|
||
|
|
def merge_facts(keep: dict, drop: dict) -> dict:
|
||
|
|
"""Merge two near-duplicate facts, keeping higher-quality fields.
|
||
|
|
|
||
|
|
The 'keep' fact is enriched with metadata from 'drop'.
|
||
|
|
"""
|
||
|
|
# Merge tags (union)
|
||
|
|
keep_tags = set(keep.get('tags', []))
|
||
|
|
drop_tags = set(drop.get('tags', []))
|
||
|
|
keep['tags'] = sorted(keep_tags | drop_tags)
|
||
|
|
|
||
|
|
# Merge related (union)
|
||
|
|
keep_related = set(keep.get('related', []))
|
||
|
|
drop_related = set(drop.get('related', []))
|
||
|
|
keep['related'] = sorted(keep_related | drop_related)
|
||
|
|
|
||
|
|
# Update source_count (sum)
|
||
|
|
keep['source_count'] = keep.get('source_count', 1) + drop.get('source_count', 1)
|
||
|
|
|
||
|
|
# Update confidence (max — we've now seen it from multiple sources)
|
||
|
|
keep['confidence'] = max(keep.get('confidence', 0), drop.get('confidence', 0))
|
||
|
|
|
||
|
|
# Track that we merged
|
||
|
|
if '_merged_from' not in keep:
|
||
|
|
keep['_merged_from'] = []
|
||
|
|
keep['_merged_from'].append(drop.get('id', 'unknown'))
|
||
|
|
|
||
|
|
return keep
|
||
|
|
|
||
|
|
|
||
|
|
def dedup_facts(
|
||
|
|
facts: List[dict],
|
||
|
|
exact_threshold: float = 1.0,
|
||
|
|
near_threshold: float = 0.95,
|
||
|
|
dry_run: bool = False,
|
||
|
|
) -> Tuple[List[dict], dict]:
|
||
|
|
"""Deduplicate a list of knowledge facts.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
facts: List of fact dicts (from index.json)
|
||
|
|
exact_threshold: Hash match = exact duplicate
|
||
|
|
near_threshold: Token similarity above this = near-duplicate
|
||
|
|
dry_run: If True, don't modify, just report
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
(deduped_facts, stats_dict)
|
||
|
|
"""
|
||
|
|
if not facts:
|
||
|
|
return [], {"total": 0, "exact_dupes": 0, "near_dupes": 0, "unique": 0}
|
||
|
|
|
||
|
|
# Phase 1: Exact dedup by content hash
|
||
|
|
hash_seen = {} # hash -> index in deduped list
|
||
|
|
exact_dupes = 0
|
||
|
|
deduped = []
|
||
|
|
|
||
|
|
for fact in facts:
|
||
|
|
text = fact.get('fact', '')
|
||
|
|
h = content_hash(text)
|
||
|
|
|
||
|
|
if h in hash_seen:
|
||
|
|
# Exact duplicate — merge metadata into existing
|
||
|
|
existing_idx = hash_seen[h]
|
||
|
|
if not dry_run:
|
||
|
|
deduped[existing_idx] = merge_facts(deduped[existing_idx], fact)
|
||
|
|
exact_dupes += 1
|
||
|
|
else:
|
||
|
|
hash_seen[h] = len(deduped)
|
||
|
|
deduped.append(fact)
|
||
|
|
|
||
|
|
# Phase 2: Near-dup by token similarity
|
||
|
|
near_dupes = 0
|
||
|
|
i = 0
|
||
|
|
while i < len(deduped):
|
||
|
|
j = i + 1
|
||
|
|
while j < len(deduped):
|
||
|
|
sim = token_similarity(deduped[i].get('fact', ''), deduped[j].get('fact', ''))
|
||
|
|
if sim >= near_threshold:
|
||
|
|
# Near-duplicate — keep higher quality
|
||
|
|
q_i = quality_score(deduped[i])
|
||
|
|
q_j = quality_score(deduped[j])
|
||
|
|
if q_i >= q_j:
|
||
|
|
if not dry_run:
|
||
|
|
deduped[i] = merge_facts(deduped[i], deduped[j])
|
||
|
|
deduped.pop(j)
|
||
|
|
else:
|
||
|
|
# j is higher quality — merge i into j, then remove i
|
||
|
|
if not dry_run:
|
||
|
|
deduped[j] = merge_facts(deduped[j], deduped[i])
|
||
|
|
deduped.pop(i)
|
||
|
|
break # i changed, restart inner loop
|
||
|
|
near_dupes += 1
|
||
|
|
else:
|
||
|
|
j += 1
|
||
|
|
i += 1
|
||
|
|
|
||
|
|
stats = {
|
||
|
|
"total": len(facts),
|
||
|
|
"exact_dupes": exact_dupes,
|
||
|
|
"near_dupes": near_dupes,
|
||
|
|
"unique": len(deduped),
|
||
|
|
"removed": len(facts) - len(deduped),
|
||
|
|
}
|
||
|
|
|
||
|
|
return deduped, stats
|
||
|
|
|
||
|
|
|
||
|
|
def dedup_index_file(
|
||
|
|
input_path: str,
|
||
|
|
output_path: Optional[str] = None,
|
||
|
|
near_threshold: float = 0.95,
|
||
|
|
dry_run: bool = False,
|
||
|
|
) -> dict:
|
||
|
|
"""Deduplicate an index.json file.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
input_path: Path to index.json
|
||
|
|
output_path: Where to write deduped file (default: overwrite input)
|
||
|
|
near_threshold: Token similarity threshold for near-dupes
|
||
|
|
dry_run: Report only, don't write
|
||
|
|
|
||
|
|
Returns stats dict.
|
||
|
|
"""
|
||
|
|
path = Path(input_path)
|
||
|
|
if not path.exists():
|
||
|
|
raise FileNotFoundError(f"Index file not found: {input_path}")
|
||
|
|
|
||
|
|
with open(path) as f:
|
||
|
|
data = json.load(f)
|
||
|
|
|
||
|
|
facts = data.get('facts', [])
|
||
|
|
deduped, stats = dedup_facts(facts, near_threshold=near_threshold, dry_run=dry_run)
|
||
|
|
|
||
|
|
if not dry_run:
|
||
|
|
data['facts'] = deduped
|
||
|
|
data['total_facts'] = len(deduped)
|
||
|
|
data['last_dedup'] = __import__('datetime').datetime.now(
|
||
|
|
__import__('datetime').timezone.utc
|
||
|
|
).isoformat()
|
||
|
|
|
||
|
|
out_path = Path(output_path) if output_path else path
|
||
|
|
with open(out_path, 'w') as f:
|
||
|
|
json.dump(data, f, indent=2, ensure_ascii=False)
|
||
|
|
|
||
|
|
return stats
|
||
|
|
|
||
|
|
|
||
|
|
def generate_test_duplicates(n: int = 20) -> List[dict]:
|
||
|
|
"""Generate test facts with intentional duplicates for testing.
|
||
|
|
|
||
|
|
Creates n unique facts plus n/4 exact dupes and n/4 near-dupes.
|
||
|
|
"""
|
||
|
|
import random
|
||
|
|
random.seed(42)
|
||
|
|
|
||
|
|
unique_facts = []
|
||
|
|
for i in range(n):
|
||
|
|
topic = random.choice(["git", "python", "docker", "rust", "nginx"])
|
||
|
|
tip = random.choice(["use verbose flags", "check logs first", "restart service", "clear cache", "update config"])
|
||
|
|
unique_facts.append({
|
||
|
|
"id": f"test:fact:{i:03d}",
|
||
|
|
"fact": f"When working with {topic}, always {tip} before deploying.",
|
||
|
|
"category": "fact",
|
||
|
|
"domain": "test",
|
||
|
|
"confidence": round(random.uniform(0.5, 1.0), 2),
|
||
|
|
"source_count": random.randint(1, 5),
|
||
|
|
"tags": [topic, "test"],
|
||
|
|
})
|
||
|
|
|
||
|
|
# Add exact duplicates (same text, different IDs)
|
||
|
|
duped = list(unique_facts)
|
||
|
|
for i in range(n // 4):
|
||
|
|
original = unique_facts[i]
|
||
|
|
dupe = dict(original)
|
||
|
|
dupe["id"] = f"test:fact:dup{i:03d}"
|
||
|
|
dupe["confidence"] = round(random.uniform(0.3, 0.8), 2)
|
||
|
|
duped.append(dupe)
|
||
|
|
|
||
|
|
# Add near-duplicates (slightly different phrasing)
|
||
|
|
for i in range(n // 4):
|
||
|
|
original = unique_facts[i]
|
||
|
|
near = dict(original)
|
||
|
|
near["id"] = f"test:fact:near{i:03d}"
|
||
|
|
near["fact"] = original["fact"].replace("always", "should").replace("before deploying", "prior to deployment")
|
||
|
|
near["confidence"] = round(random.uniform(0.4, 0.9), 2)
|
||
|
|
duped.append(near)
|
||
|
|
|
||
|
|
return duped
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser(description="Knowledge deduplication")
|
||
|
|
parser.add_argument("--input", help="Path to index.json")
|
||
|
|
parser.add_argument("--output", help="Output path (default: overwrite input)")
|
||
|
|
parser.add_argument("--threshold", type=float, default=0.95,
|
||
|
|
help="Near-dup similarity threshold (default: 0.95)")
|
||
|
|
parser.add_argument("--dry-run", action="store_true", help="Report only, don't write")
|
||
|
|
parser.add_argument("--test", action="store_true", help="Run built-in dedup test")
|
||
|
|
parser.add_argument("--json", action="store_true", help="JSON output")
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
if args.test:
|
||
|
|
test_facts = generate_test_duplicates(20)
|
||
|
|
print(f"Generated {len(test_facts)} test facts (20 unique + dupes)")
|
||
|
|
deduped, stats = dedup_facts(test_facts, near_threshold=args.threshold)
|
||
|
|
print(f"\nDedup results:")
|
||
|
|
print(f" Total input: {stats['total']}")
|
||
|
|
print(f" Exact dupes: {stats['exact_dupes']}")
|
||
|
|
print(f" Near dupes: {stats['near_dupes']}")
|
||
|
|
print(f" Unique output: {stats['unique']}")
|
||
|
|
print(f" Removed: {stats['removed']}")
|
||
|
|
|
||
|
|
# Verify: should have ~20 unique (some merged)
|
||
|
|
assert stats['unique'] <= 20, f"Too many unique: {stats['unique']} > 20"
|
||
|
|
assert stats['unique'] >= 15, f"Too few unique: {stats['unique']} < 15"
|
||
|
|
assert stats['removed'] > 0, "No duplicates removed"
|
||
|
|
print("\nOK: Dedup test passed")
|
||
|
|
return
|
||
|
|
|
||
|
|
if not args.input:
|
||
|
|
print("ERROR: Provide --input or --test")
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
stats = dedup_index_file(args.input, args.output, args.threshold, args.dry_run)
|
||
|
|
|
||
|
|
if args.json:
|
||
|
|
print(json.dumps(stats, indent=2))
|
||
|
|
else:
|
||
|
|
print(f"Dedup results:")
|
||
|
|
print(f" Total input: {stats['total']}")
|
||
|
|
print(f" Exact dupes: {stats['exact_dupes']}")
|
||
|
|
print(f" Near dupes: {stats['near_dupes']}")
|
||
|
|
print(f" Unique output: {stats['unique']}")
|
||
|
|
print(f" Removed: {stats['removed']}")
|
||
|
|
if args.dry_run:
|
||
|
|
print(" (dry run — no changes written)")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|