285 lines
10 KiB
Python
Executable File
285 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
decay_scanner.py
|
|
Selective archiving script for low-importance AI-generated chunks.
|
|
Runs via weekly cron (0 3 * * 0).
|
|
|
|
Rules:
|
|
- source_type in ["human", "procedural"] → exempt (never archive)
|
|
- importance_score >= 0.7 → exempt
|
|
- archived == True → skip (already archived)
|
|
- half_life: 90d if importance_score >= 0.3, else 30d
|
|
- decay_score < 0.1:
|
|
- If confidence_score >= 0.7 → alert (report, don't archive)
|
|
- Otherwise → archive (archived = True)
|
|
- gabi_* collections are completely ignored
|
|
|
|
Usage:
|
|
python3 decay_scanner.py [--collection knowledge_base_hybrid] [--dry-run]
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import math
|
|
import argparse
|
|
import requests
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
# ─── Config ────────────────────────────────────────────────────────────────
|
|
QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333")
|
|
COLLECTION = os.environ.get("QDRANT_COLLECTION", "knowledge_base")
|
|
SCROLL_LIMIT = 100 # Qdrant pagination
|
|
LOG_DIR = Path(os.environ.get("HERMES_LOGS_DIR", str(Path.home() / ".hermes" / "logs")))
|
|
LOG_FILE = LOG_DIR / "decay_scanner.log"
|
|
|
|
# ─── Helpers ──────────────────────────────────────────────────────────────
|
|
|
|
def now_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def calculate_decay_score(last_accessed_at: str, importance_score: float) -> float:
|
|
"""
|
|
Calculate exponential decay: score = exp(-ln(2) * age_days / half_life).
|
|
More important chunks persist longer (larger half-lives).
|
|
"""
|
|
try:
|
|
last = datetime.fromisoformat(last_accessed_at.replace("Z", "+00:00"))
|
|
except (ValueError, TypeError):
|
|
# If timestamp is invalid, assume now (hasn't decayed yet)
|
|
return 1.0
|
|
|
|
now = datetime.now(timezone.utc)
|
|
age_days = max(0, (now - last).total_seconds() / 86400)
|
|
|
|
# Fix: LARGER half-life for more important chunks
|
|
if importance_score >= 0.3:
|
|
half_life = 90 # medium/high chunks → 90 days
|
|
else:
|
|
half_life = 30 # low chunks → 30 days
|
|
|
|
decay_score = math.exp(-math.log(2) * age_days / half_life)
|
|
return decay_score
|
|
|
|
|
|
def ensure_log_dir():
|
|
"""Create log directory if it doesn't exist."""
|
|
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def log_message(msg: str):
|
|
"""Log to stdout and append to log file."""
|
|
ts = now_iso()
|
|
line = f"[{ts}] {msg}"
|
|
print(line)
|
|
ensure_log_dir()
|
|
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
|
f.write(line + "\n")
|
|
|
|
|
|
# ─── Qdrant Operations ────────────────────────────────────────────────────
|
|
|
|
def scroll_chunks(collection: str, limit: int = SCROLL_LIMIT):
|
|
"""
|
|
Generator that iterates over all points in the collection via scroll.
|
|
Avoids loading the entire collection into memory.
|
|
"""
|
|
offset = None
|
|
total_scanned = 0
|
|
|
|
while True:
|
|
payload = {
|
|
"limit": limit,
|
|
"with_payload": True,
|
|
"with_vector": False,
|
|
}
|
|
if offset is not None:
|
|
payload["offset"] = offset
|
|
|
|
try:
|
|
resp = requests.post(
|
|
f"{QDRANT_URL}/collections/{collection}/points/scroll",
|
|
headers={"Content-Type": "application/json"},
|
|
json=payload,
|
|
timeout=30,
|
|
)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
result = data.get("result", {})
|
|
points = result.get("points", [])
|
|
|
|
if not points:
|
|
break
|
|
|
|
for point in points:
|
|
yield point
|
|
total_scanned += 1
|
|
|
|
offset = result.get("next_page_offset")
|
|
if offset is None:
|
|
break
|
|
|
|
except Exception as e:
|
|
log_message(f"❌ Qdrant scroll error: {e}")
|
|
break
|
|
|
|
log_message(f"📊 Total chunks scanned: {total_scanned}")
|
|
|
|
|
|
def update_point_archived(point_id: str, collection: str, decay_score: float, dry_run: bool = False):
|
|
"""Update point payload: archived=True + calculated decay_score."""
|
|
if dry_run:
|
|
log_message(f" [DRY-RUN] Would archive point {point_id} (decay_score={decay_score:.4f})")
|
|
return True
|
|
|
|
try:
|
|
resp = requests.post(
|
|
f"{QDRANT_URL}/collections/{collection}/points/payload",
|
|
headers={"Content-Type": "application/json"},
|
|
json={
|
|
"points": [point_id],
|
|
"payload": {
|
|
"archived": True,
|
|
"decay_score": decay_score,
|
|
},
|
|
},
|
|
timeout=10,
|
|
)
|
|
resp.raise_for_status()
|
|
return True
|
|
except Exception as e:
|
|
log_message(f" ❌ Failed to archive point {point_id}: {e}")
|
|
return False
|
|
|
|
|
|
# ─── Main ─────────────────────────────────────────────────────────────────
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Decay Scanner — Selective chunk archiving")
|
|
parser.add_argument("--collection", default=COLLECTION, help="Qdrant collection name")
|
|
parser.add_argument("--dry-run", action="store_true", help="Simulation — does not modify anything")
|
|
parser.add_argument("--threshold", type=float, default=0.1, help="Decay threshold for archiving")
|
|
args = parser.parse_args()
|
|
|
|
collection = args.collection
|
|
|
|
# Ignore gabi_* collections
|
|
if collection.startswith("gabi_"):
|
|
log_message(f"⏭️ Collection '{collection}' is exempt (gabi_*). Exiting.")
|
|
return
|
|
|
|
log_message(f"🚀 Starting decay scanner (collection={collection}, threshold={args.threshold}, dry_run={args.dry_run})")
|
|
|
|
# Metrics
|
|
stats = {
|
|
"scanned": 0,
|
|
"archived": 0,
|
|
"alerted": 0,
|
|
"skipped_human": 0,
|
|
"skipped_procedural": 0,
|
|
"skipped_high_importance": 0,
|
|
"skipped_already_archived": 0,
|
|
"failed": 0,
|
|
}
|
|
|
|
alerts = [] # List of alerts (decay < threshold but confidence >= 0.7)
|
|
|
|
for point in scroll_chunks(collection):
|
|
stats["scanned"] += 1
|
|
|
|
point_id = point.get("id")
|
|
payload = point.get("payload", {})
|
|
|
|
source_type = payload.get("source_type", "unknown")
|
|
importance_score = payload.get("importance_score", 0.5)
|
|
archived = payload.get("archived", False)
|
|
last_accessed_at = payload.get("last_accessed_at", payload.get("created_at", now_iso()))
|
|
confidence_score = payload.get("confidence_score", 1.0)
|
|
|
|
# Skip: already archived
|
|
if archived:
|
|
stats["skipped_already_archived"] += 1
|
|
continue
|
|
|
|
# Skip: human (exempt)
|
|
if source_type == "human":
|
|
stats["skipped_human"] += 1
|
|
continue
|
|
|
|
# Skip: procedural (exempt)
|
|
if source_type == "procedural":
|
|
stats["skipped_procedural"] += 1
|
|
continue
|
|
|
|
# Skip: high importance
|
|
if importance_score >= 0.7:
|
|
stats["skipped_high_importance"] += 1
|
|
continue
|
|
|
|
# Calculate decay
|
|
decay_score = calculate_decay_score(last_accessed_at, importance_score)
|
|
|
|
# Check threshold
|
|
if decay_score < args.threshold:
|
|
# Decay-confidence rule: if confidence is high, alert instead of archiving
|
|
if confidence_score >= 0.7:
|
|
stats["alerted"] += 1
|
|
alerts.append({
|
|
"point_id": point_id,
|
|
"decay_score": round(decay_score, 4),
|
|
"confidence_score": round(confidence_score, 2),
|
|
"importance_score": round(importance_score, 2),
|
|
"age_days": round((datetime.now(timezone.utc) - datetime.fromisoformat(last_accessed_at.replace("Z", "+00:00"))).total_seconds() / 86400, 1),
|
|
"reason": "decay < threshold but confidence >= 0.7 — manual review recommended",
|
|
})
|
|
log_message(f" ⚠️ ALERT: point {point_id} (decay={decay_score:.4f}, confidence={confidence_score:.2f}) — manual review recommended")
|
|
else:
|
|
# Archive
|
|
ok = update_point_archived(point_id, collection, decay_score, args.dry_run)
|
|
if ok:
|
|
stats["archived"] += 1
|
|
log_message(f" 📦 Archived: point {point_id} (decay={decay_score:.4f}, importance={importance_score:.2f})")
|
|
else:
|
|
stats["failed"] += 1
|
|
|
|
# Structured JSON report
|
|
report = {
|
|
"timestamp": now_iso(),
|
|
"collection": collection,
|
|
"threshold": args.threshold,
|
|
"dry_run": args.dry_run,
|
|
"scanned": stats["scanned"],
|
|
"archived": stats["archived"],
|
|
"alerted": stats["alerted"],
|
|
"skipped_human": stats["skipped_human"],
|
|
"skipped_procedural": stats["skipped_procedural"],
|
|
"skipped_high_importance": stats["skipped_high_importance"],
|
|
"skipped_already_archived": stats["skipped_already_archived"],
|
|
"failed": stats["failed"],
|
|
"alerts": alerts,
|
|
}
|
|
|
|
log_message("=" * 60)
|
|
log_message("📊 DECAY SCANNER REPORT")
|
|
log_message("=" * 60)
|
|
log_message(f" Scanned: {stats['scanned']}")
|
|
log_message(f" Archived: {stats['archived']}")
|
|
log_message(f" Alerts (decay+conf.): {stats['alerted']}")
|
|
log_message(f" Skipped human: {stats['skipped_human']}")
|
|
log_message(f" Skipped procedural: {stats['skipped_procedural']}")
|
|
log_message(f" Skipped high imp.: {stats['skipped_high_importance']}")
|
|
log_message(f" Skipped archived: {stats['skipped_already_archived']}")
|
|
log_message(f" Failures: {stats['failed']}")
|
|
log_message("=" * 60)
|
|
|
|
# JSON report to stderr (parseable)
|
|
print(json.dumps(report, ensure_ascii=False, indent=2), file=sys.stderr)
|
|
|
|
log_message("✅ Decay scanner complete.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|