memoryweave/scripts/wiki_curator.py

#!/usr/bin/env python3
"""
Wiki Curator for 织忆 (MemoryWeave) — Auto knowledge curation pipeline.

Scans .md files, extracts concepts/entities/relations, writes to 织忆 via API.
Usage:
  python3 %(script)s                # incremental (SHA-256 diff tracked)
  python3 %(script)s --dry-run       # preview only
  python3 %(script)s --force         # re-process all files
  python3 %(script)s --dir PATH      # scan custom directory
"""

import hashlib, json, os, re, sys, time
from pathlib import Path

try:
    import requests
except ImportError:
    print("ERROR: requests not installed. Run: uv pip install requests")
    sys.exit(1)

# ── Config ──
ZHIYI_API = "http://localhost:7821"
ZHIYI_KEY = "zhiyi-dev-key-2026"
STATE_FILE = Path.home() / ".hermes" / "wiki_curator_state.json"
HEADERS = {"X-API-Key": ZHIYI_KEY, "Content-Type": "application/json"}

EXCLUDE_DIRS = frozenset({
    "__pycache__", ".git", ".obsidian", ".trash", "node_modules",
    "backups", ".cache", ".venv", ".npm-global",
})

MIN_FILE_CHARS = 500

def _sha256(text: str) -> str:
    return hashlib.sha256(text.encode()).hexdigest()

def _load_state() -> dict:
    if STATE_FILE.exists():
        return json.loads(STATE_FILE.read_text())
    return {}

def _save_state(state: dict):
    STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
    STATE_FILE.write_text(json.dumps(state, indent=2, ensure_ascii=False))

def _scan_files(root: Path):
    for path in root.rglob("*.md"):
        if any(excl in path.parts for excl in EXCLUDE_DIRS):
            continue
        if path.name.startswith("_"):
            continue
        yield path

def _extract(text: str, filename: str):
    concepts = []
    entities = []
    relations = []

    # Headings → concepts
    for m in re.finditer(r"^#{2,3}\s+(.+)", text, re.MULTILINE):
        name = m.group(1).strip()
        if len(name) > 3:
            concepts.append({"name": name, "source": filename})

    # Bold phrases → entities
    for m in re.finditer(r"\*\*(.+?)\*\*", text):
        name = m.group(1).strip()
        if len(name) > 2 and len(concepts + entities) < 30:
            entities.append({"name": name, "source": filename})

    # First sentence of each paragraph as relation hint
    for m in re.finditer(r"^([^#\n][^。\n]{10,}。[^。\n]*)", text, re.MULTILINE):
        sentence = m.group(1).strip()
        if len(relations) >= 10:
            break
        relations.append({"text": sentence[:200], "source": filename})

    return concepts, entities, relations

def run(dry_run=False, force=False, root_dir=None):
    start = time.time()
    root = Path(root_dir).expanduser() if root_dir else Path.home() / "mc"
    if not root.exists():
        print(f"ERROR: directory not found: {root}")
        return 1

    state = _load_state() if not force else {}
    files_processed = 0
    total_concepts = 0
    total_entities = 0
    total_relations = 0

    for path in _scan_files(root):
        try:
            content = path.read_text()
        except Exception:
            continue

        if len(content) < MIN_FILE_CHARS:
            continue

        sha = _sha256(content)
        rel_path = str(path.relative_to(root))
        if not force and rel_path in state and state[rel_path] == sha:
            continue

        concepts, entities, relations = _extract(content, path.name)
        files_processed += 1

        if dry_run:
            total_concepts += len(concepts)
            total_entities += len(entities)
            total_relations += len(relations)
            state[rel_path] = sha
            continue

        # Write to 织忆
        for c in concepts:
            try:
                payload = {
                    "agent_id": "wiki-curator",
                    "content": f"## {c['name']}\nFrom: {c['source']}",
                    "category": "wiki",
                    "metadata": {"source": rel_path, "concept_type": "concept"},
                }
                requests.post(f"{ZHIYI_API}/api/v1/commit", json=payload, headers=HEADERS, timeout=5)
                total_concepts += 1
            except Exception:
                pass
            time.sleep(0.1)

        for e in entities:
            try:
                payload = {
                    "agent_id": "wiki-curator",
                    "content": f"Entity: {e['name']} (from {e['source']})",
                    "category": "wiki",
                    "metadata": {"source": rel_path, "concept_type": "entity"},
                }
                requests.post(f"{ZHIYI_API}/api/v1/commit", json=payload, headers=HEADERS, timeout=5)
                total_entities += 1
            except Exception:
                pass
            time.sleep(0.1)

        for r in relations:
            try:
                payload = {
                    "from": path.stem[:50], "to": r["text"][:50],
                    "relation": "MENTIONS", "namespace": "wiki",
                }
                requests.post(f"{ZHIYI_API}/api/v1/graph/edge", json=payload, headers=HEADERS, timeout=5)
                total_relations += 1
            except Exception:
                pass
            time.sleep(0.1)

        state[rel_path] = sha

        if files_processed % 10 == 0:
            print(f"  ... {files_processed} files processed", file=sys.stderr)

    if not dry_run:
        _save_state(state)

    elapsed = time.time() - start
    print(f"{'='*60}")
    print(f"  📊 处理总结")
    print(f"  处理文件数:   {files_processed}")
    print(f"  概念写入数:   {total_concepts}")
    print(f"  实体写入数:   {total_entities}")
    print(f"  关系写入数:   {total_relations}")
    print(f"  耗时:         {elapsed:.1f}s")
    print(f"{'='*60}")
    print()
    print("WIKI_CURATOR_OK")
    return 0

if __name__ == "__main__":
    dry_run = "--dry-run" in sys.argv
    force = "--force" in sys.argv
    root_dir = None
    if "--dir" in sys.argv:
        idx = sys.argv.index("--dir")
        if idx + 1 < len(sys.argv):
            root_dir = sys.argv[idx + 1]
    sys.exit(run(dry_run=dry_run, force=force, root_dir=root_dir))