311 lines
11 KiB
Python
Executable File
311 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
semantic_dedup.py
|
|
Scanner mensal de near-duplicates no knowledge_base via Qdrant HNSW index.
|
|
Rodo no primeiro domingo de cada mês (cron: 0 3 1 * *).
|
|
|
|
Regras:
|
|
- Ignora coleções com prefixo em DEDUP_EXEMPT_PREFIXES (csv)
|
|
- Não deleta automaticamente — apenas emite relatório JSON de candidatos
|
|
- Threshold de similaridade: 0.92 (configurável)
|
|
- Usa query_batch_points() para delegar busca de vizinhos ao Qdrant
|
|
(HNSW index nativo, O(n/batch) requests em vez de O(n²) brute-force)
|
|
|
|
Uso:
|
|
python3 semantic_dedup.py [--collection knowledge_base] [--threshold 0.92] [--dry-run]
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import math
|
|
import argparse
|
|
import requests
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import List, Dict, Optional
|
|
|
|
try:
|
|
from qdrant_client import QdrantClient, models
|
|
_has_qdrant_client = True
|
|
except ImportError:
|
|
_has_qdrant_client = False
|
|
|
|
# ─── Config ────────────────────────────────────────────────────────────────
|
|
QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333")
|
|
COLLECTION = os.environ.get("QDRANT_COLLECTION", "knowledge_base")
|
|
SCROLL_LIMIT = 50 # paginação Qdrant (evita timeout em coleções grandes)
|
|
SIMILARITY_THRESHOLD = 0.92
|
|
TOP_NEIGHBORS = 10
|
|
|
|
LOG_DIR = Path.home() / ".hermes" / "logs"
|
|
LOG_FILE = LOG_DIR / "semantic_dedup.log"
|
|
REPORT_FILE = LOG_DIR / "semantic_dedup_report.json"
|
|
|
|
|
|
def now_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def log_message(msg: str):
|
|
ts = now_iso()
|
|
line = f"[{ts}] {msg}"
|
|
print(line)
|
|
try:
|
|
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
|
f.write(line + "\n")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ─── Qdrant Operations ────────────────────────────────────────────────────
|
|
|
|
def scroll_all_chunks(collection: str) -> List[Dict]:
|
|
"""
|
|
Carrega todos os pontos da coleção paginando via scroll.
|
|
Retorna lista de {id, vector, payload}.
|
|
"""
|
|
all_chunks = []
|
|
offset = None
|
|
scanned = 0
|
|
|
|
while True:
|
|
payload = {
|
|
"limit": SCROLL_LIMIT,
|
|
"with_payload": True,
|
|
"with_vector": True,
|
|
}
|
|
if offset is not None:
|
|
payload["offset"] = offset
|
|
|
|
try:
|
|
resp = requests.post(
|
|
f"{QDRANT_URL}/collections/{collection}/points/scroll",
|
|
headers={"Content-Type": "application/json"},
|
|
json=payload,
|
|
timeout=30,
|
|
)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
result = data.get("result", {})
|
|
points = result.get("points", [])
|
|
|
|
if not points:
|
|
break
|
|
|
|
for point in points:
|
|
# Pegar apenas vetor dense para similarity
|
|
vector = point.get("vector")
|
|
dense = None
|
|
if isinstance(vector, dict):
|
|
dense = vector.get("dense")
|
|
elif isinstance(vector, list):
|
|
dense = vector # fallback: vetor simples
|
|
|
|
if dense:
|
|
all_chunks.append({
|
|
"id": point.get("id"),
|
|
"vector": dense,
|
|
"payload": point.get("payload", {}),
|
|
})
|
|
|
|
scanned += len(points)
|
|
offset = result.get("next_page_offset")
|
|
if offset is None:
|
|
break
|
|
|
|
except Exception as e:
|
|
log_message(f"❌ Erro no scroll Qdrant: {e}")
|
|
break
|
|
|
|
log_message(f"📊 Total chunks carregados: {len(all_chunks)} / {scanned} escaneados")
|
|
return all_chunks
|
|
|
|
|
|
def cosine_similarity(v1: List[float], v2: List[float]) -> float:
|
|
"""Calcula cosine similarity entre dois vetores."""
|
|
if len(v1) != len(v2):
|
|
return 0.0
|
|
|
|
dot = sum(a * b for a, b in zip(v1, v2))
|
|
norm1 = math.sqrt(sum(a * a for a in v1))
|
|
norm2 = math.sqrt(sum(b * b for b in v2))
|
|
|
|
if norm1 == 0 or norm2 == 0:
|
|
return 0.0
|
|
|
|
return dot / (norm1 * norm2)
|
|
|
|
|
|
def find_near_duplicates(chunks: List[Dict],
|
|
threshold: float = SIMILARITY_THRESHOLD,
|
|
collection: str = COLLECTION) -> List[Dict]:
|
|
"""
|
|
Encontra pares de near-duplicates via Qdrant query_batch_points.
|
|
Delega a busca de vizinhos ao índice HNSW nativo do Qdrant —
|
|
O(n/batch) HTTP requests em vez de O(n²) brute-force local.
|
|
|
|
Cada ponto consulta o Qdrant pelo vizinho mais próximo acima de
|
|
threshold, excluindo self-match via HasIdCondition no servidor.
|
|
"""
|
|
n = len(chunks)
|
|
if n < 2:
|
|
return []
|
|
|
|
if not _has_qdrant_client:
|
|
log_message("⚠️ qdrant-client não instalado. Instale com: pip install qdrant-client")
|
|
log_message(" Fallback não disponível — pulando dedup.")
|
|
return []
|
|
|
|
client = QdrantClient(url=QDRANT_URL)
|
|
BATCH_SIZE = max(1, min(200, n // 10 + 1)) # escala com tamanho da coleção
|
|
candidates = []
|
|
ids_seen = set()
|
|
|
|
for batch_start in range(0, n, BATCH_SIZE):
|
|
batch = chunks[batch_start:batch_start + BATCH_SIZE]
|
|
|
|
requests_list = []
|
|
for p in batch:
|
|
point_id = p["id"]
|
|
vec = p.get("vector")
|
|
if vec is None:
|
|
continue
|
|
requests_list.append(
|
|
models.QueryRequest(
|
|
query=vec,
|
|
using="dense",
|
|
limit=1,
|
|
score_threshold=threshold,
|
|
filter=models.Filter(
|
|
must_not=[
|
|
models.HasIdCondition(has_id=[point_id])
|
|
]
|
|
),
|
|
with_payload=True,
|
|
with_vector=False,
|
|
)
|
|
)
|
|
|
|
if not requests_list:
|
|
continue
|
|
|
|
try:
|
|
results = client.query_batch_points(
|
|
collection_name=collection,
|
|
requests=requests_list,
|
|
)
|
|
except Exception as e:
|
|
log_message(f"❌ Erro query_batch_points (batch {batch_start}): {e}")
|
|
continue
|
|
|
|
# results is list[QueryResponse]; each has .points → list[ScoredPoint]
|
|
for p, query_resp in zip(batch, results):
|
|
scored_points = query_resp.points if hasattr(query_resp, 'points') else []
|
|
if not scored_points:
|
|
continue
|
|
best = scored_points[0]
|
|
pair_key = tuple(sorted([str(p["id"]), str(best.id)]))
|
|
if pair_key in ids_seen:
|
|
continue
|
|
ids_seen.add(pair_key)
|
|
candidates.append({
|
|
"chunk_id_a": str(p["id"]),
|
|
"chunk_id_b": str(best.id),
|
|
"similarity": round(best.score, 6),
|
|
"source_a": p["payload"].get("source", "unknown"),
|
|
"source_b": (best.payload or {}).get("source", "unknown"),
|
|
"title_a": p["payload"].get("title", "")[:60],
|
|
"title_b": (best.payload or {}).get("title", "")[:60],
|
|
"text_preview_a": p["payload"].get("text", "")[:100],
|
|
"text_preview_b": (best.payload or {}).get("text", "")[:100],
|
|
})
|
|
|
|
# Ordenar por similaridade decrescente
|
|
candidates.sort(key=lambda x: x["similarity"], reverse=True)
|
|
return candidates
|
|
|
|
|
|
def generate_report(candidates: List[Dict], collection: str, threshold: float, scanned: int) -> Dict:
|
|
"""Gera relatório estruturado em JSON."""
|
|
return {
|
|
"timestamp": now_iso(),
|
|
"collection": collection,
|
|
"threshold": threshold,
|
|
"scanned_chunks": scanned,
|
|
"near_duplicate_pairs": len(candidates),
|
|
"candidates": candidates,
|
|
"recommendation": (
|
|
f"{len(candidates)} pares de near-duplicates encontrados. "
|
|
"Revisar manualmente e aplicar merge via Qdrant point update se aprovado."
|
|
),
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Semantic Dedup Scanner")
|
|
parser.add_argument("--collection", default=COLLECTION, help="Nome da coleção Qdrant")
|
|
parser.add_argument("--threshold", type=float, default=SIMILARITY_THRESHOLD, help="Threshold cosine similarity")
|
|
parser.add_argument("--dry-run", action="store_true", help="Só escaneia, não salva relatório")
|
|
args = parser.parse_args()
|
|
|
|
collection = args.collection
|
|
|
|
# Ignorar coleções com prefixos exempt (via DEDUP_EXEMPT_PREFIXES env var)
|
|
exempt_prefixes = os.environ.get("DEDUP_EXEMPT_PREFIXES", "").split(",")
|
|
exempt_prefixes = [p.strip() for p in exempt_prefixes if p.strip()]
|
|
for prefix in exempt_prefixes:
|
|
if collection.startswith(prefix):
|
|
log_message(f"⏭️ Coleção '{collection}' é exempt (prefixo '{prefix}'). Saindo.")
|
|
return
|
|
|
|
log_message(f"🚀 Iniciando semantic dedup (collection={collection}, threshold={args.threshold}, dry_run={args.dry_run})")
|
|
|
|
# Carregar chunks
|
|
chunks = scroll_all_chunks(collection)
|
|
|
|
if not chunks:
|
|
log_message("⚠️ Nenhum chunk encontrado na coleção.")
|
|
return
|
|
|
|
# Encontrar near-duplicates
|
|
log_message(f"🔍 Analisando similaridade entre {len(chunks)} chunks...")
|
|
candidates = find_near_duplicates(chunks, threshold=args.threshold)
|
|
|
|
# Gerar relatório
|
|
report = generate_report(candidates, collection, args.threshold, len(chunks))
|
|
|
|
log_message("=" * 60)
|
|
log_message("📊 RELATÓRIO SEMANTIC DEDUP")
|
|
log_message("=" * 60)
|
|
log_message(f" Chunks escaneados: {report['scanned_chunks']}")
|
|
log_message(f" Near-duplicate pairs: {report['near_duplicate_pairs']}")
|
|
|
|
if candidates:
|
|
log_message(f" Top similaridade: {candidates[0]['similarity']:.4f}")
|
|
log_message(f" Top par: {candidates[0]['chunk_id_a']} ↔ {candidates[0]['chunk_id_b']}")
|
|
else:
|
|
log_message(" Nenhum near-duplicate encontrado.")
|
|
|
|
log_message("=" * 60)
|
|
|
|
# Salvar relatório JSON
|
|
if not args.dry_run and candidates:
|
|
try:
|
|
REPORT_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(REPORT_FILE, "w", encoding="utf-8") as f:
|
|
json.dump(report, f, ensure_ascii=False, indent=2)
|
|
log_message(f"📄 Relatório salvo: {REPORT_FILE}")
|
|
except Exception as e:
|
|
log_message(f"❌ Erro ao salvar relatório: {e}")
|
|
|
|
# Output JSON para stderr (parseável)
|
|
print(json.dumps(report, ensure_ascii=False, indent=2), file=sys.stderr)
|
|
|
|
log_message("✅ Semantic dedup completo.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|