fix: resolve 5 easy Phase 4 performance/resilience issues
V4.04: Add retry_transient() to dlq_manager — re-enables reported transient failures for retry with --retry CLI flag, increments retry_count/last_retry V4.05: save_report() now truncates JSONL to MAX_REPORT_HISTORY entries after each append (best-effort rotation) V4.06: Wrap datetime.fromisoformat() in try/except (ValueError, TypeError) in get_status_summary() — skips entries with malformed timestamps V4.08: Replace body[:50] dedup key with hashlib.sha256(body).hexdigest() in fabric-retrieve.py deduplicate() V4.09: Add OrderedDict LRU cache (256 entries) to embedding.py get_embedding() — avoids recomputing embeddings for repeated text
This commit is contained in:
parent
e5edb58a19
commit
1422a6d583
|
|
@ -1,10 +1,11 @@
|
|||
"""
|
||||
Embedding client. Provider-agnostic — defaults to OpenRouter, configurable
|
||||
via EMBEDDING_API_BASE and EMBEDDING_MODEL for local Ollama/vLLM/llama.cpp.
|
||||
Mandatory dimension validation.
|
||||
Mandatory dimension validation with in-memory LRU cache.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from collections import OrderedDict
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -20,12 +21,21 @@ EMBEDDING_MODEL = os.environ.get(
|
|||
"EMBEDDING_MODEL", "qwen/qwen3-embedding-8b"
|
||||
)
|
||||
|
||||
# In-memory LRU cache — avoids recomputing embeddings for repeated text
|
||||
_EMBEDDING_CACHE_MAXSIZE = 256
|
||||
_embedding_cache: OrderedDict = OrderedDict()
|
||||
|
||||
|
||||
async def get_embedding(text: str) -> list[float]:
|
||||
"""
|
||||
Generates embedding via the configured backend.
|
||||
Validates that the returned dimensions match EMBEDDING_DIMS.
|
||||
Results are cached in-memory (LRU, max 256 entries).
|
||||
"""
|
||||
# Check cache
|
||||
if text in _embedding_cache:
|
||||
_embedding_cache.move_to_end(text)
|
||||
return _embedding_cache[text]
|
||||
headers = {"Content-Type": "application/json"}
|
||||
|
||||
if "openrouter" in EMBEDDING_API_BASE.lower():
|
||||
|
|
@ -64,4 +74,10 @@ async def get_embedding(text: str) -> list[float]:
|
|||
)
|
||||
|
||||
logger.debug(f"Embedding generated: {len(vec)} dims")
|
||||
|
||||
# Store in LRU cache
|
||||
_embedding_cache[text] = vec
|
||||
if len(_embedding_cache) > _EMBEDDING_CACHE_MAXSIZE:
|
||||
_embedding_cache.popitem(last=False)
|
||||
|
||||
return vec
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ Usage:
|
|||
python3 fabric-retrieve.py "auth module" --agent icarus --project myapp
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
|
|
@ -262,7 +264,8 @@ def deduplicate(entries):
|
|||
seen = {}
|
||||
result = []
|
||||
for e in entries:
|
||||
key = (e.get("agent", ""), e.get("type", ""), e.get("_body", "")[:50])
|
||||
body = e.get("_body", "") or ""
|
||||
key = hashlib.sha256(body.encode()).hexdigest()
|
||||
existing = seen.get(key)
|
||||
if existing:
|
||||
# Keep the newer one
|
||||
|
|
|
|||
|
|
@ -82,6 +82,19 @@ def compute_error_hash(file: str, error: str) -> str:
|
|||
import hashlib
|
||||
return hashlib.md5(f"{file}:{error[:80]}".encode()).hexdigest()[:8]
|
||||
|
||||
# ─── Retry ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def retry_transient(entries: List[DLQEntry]) -> int:
|
||||
"""Re-enable transient failures for retry. Returns count of re-enabled entries."""
|
||||
count = 0
|
||||
for e in entries:
|
||||
if e.failure_class == "transient" and e.reported:
|
||||
e.reported = False
|
||||
e.retry_count += 1
|
||||
e.last_retry = datetime.now().isoformat()
|
||||
count += 1
|
||||
return count
|
||||
|
||||
# ─── Reporting ─────────────────────────────────────────────────────────────
|
||||
|
||||
def build_report(entries: List[DLQEntry]) -> Dict:
|
||||
|
|
@ -135,11 +148,23 @@ def save_report(report: Dict):
|
|||
os.makedirs(REPORT_DIR, exist_ok=True)
|
||||
timestamp = datetime.now().isoformat()
|
||||
|
||||
# JSONL
|
||||
# JSONL — append then truncate to MAX_REPORT_HISTORY
|
||||
with open(REPORT_LOG, "a") as f:
|
||||
f.write(json.dumps({"timestamp": timestamp, **report}, ensure_ascii=False) + "\n")
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
|
||||
# Rotate: keep only the last MAX_REPORT_HISTORY entries
|
||||
try:
|
||||
with open(REPORT_LOG, "r") as f:
|
||||
lines = f.readlines()
|
||||
if len(lines) > MAX_REPORT_HISTORY:
|
||||
with open(REPORT_LOG, "w") as f:
|
||||
f.writelines(lines[-MAX_REPORT_HISTORY:])
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
except OSError:
|
||||
pass # best-effort rotation
|
||||
|
||||
def mark_reported(entries: List[DLQEntry]):
|
||||
for e in entries:
|
||||
|
|
@ -149,7 +174,16 @@ def get_status_summary(entries: List[DLQEntry]) -> Dict:
|
|||
total = len(entries)
|
||||
unreported = len([e for e in entries if not e.reported])
|
||||
by_class = Counter(e.failure_class for e in entries)
|
||||
recent = [e for e in entries if datetime.now() - datetime.fromisoformat(e.timestamp.replace("Z", "+00:00")) < timedelta(hours=24)]
|
||||
# Parse timestamps with fallback for malformed values
|
||||
now = datetime.now()
|
||||
recent = []
|
||||
for e in entries:
|
||||
try:
|
||||
ts = datetime.fromisoformat(e.timestamp.replace("Z", "+00:00"))
|
||||
except (ValueError, TypeError):
|
||||
continue # skip entries with unparseable timestamps
|
||||
if now - ts < timedelta(hours=24):
|
||||
recent.append(e)
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
|
|
@ -168,10 +202,21 @@ def main():
|
|||
p.add_argument("--status", action="store_true", help="Status resumido")
|
||||
p.add_argument("--json", action="store_true", help="Saída JSON")
|
||||
p.add_argument("--silent-if-ok", action="store_true", help="Silencioso se DLQ ok")
|
||||
p.add_argument("--retry", action="store_true", help="Re-enable transient failures for retry")
|
||||
args = p.parse_args()
|
||||
|
||||
entries = load_dlq()
|
||||
|
||||
if args.retry:
|
||||
# Classify first so transient detection works
|
||||
for e in entries:
|
||||
if e.failure_class == "unknown":
|
||||
e.failure_class = classify_error(e.error)
|
||||
count = retry_transient(entries)
|
||||
save_dlq(entries)
|
||||
print(f"✅ {count} falha(s) transiente(s) re-habilitadas para retry")
|
||||
return
|
||||
|
||||
if args.status:
|
||||
summary = get_status_summary(entries)
|
||||
if args.json:
|
||||
|
|
|
|||
Loading…
Reference in New Issue