#!/usr/bin/env python3
"""
Wiki Curator — 自动知识策展管线 (P5)

把 Obsidian 笔记里的知识点抽取成织忆记忆 + 图谱边。

2026-09-11 加固背景：一次干跑 163 篇笔记 → 拟写 概念5435 + 实体8647 + 关系8638，
而织忆整库当时仅 18629 个节点（一口气灌大 ~75%）。抽样质量更致命——它把设计
文档里的「提示词片段」当成实体灌进去（「只提取明确信息，不要推测」「纯闲聊→IS=0」
「facts 是"牧尘说了什么"」）。典型病症：把「关于提取的说明书」当成「提取结果」。
→ 因此该管线必须先过四道闸门，缺一不开。

四道闸门
--------
① 选择性  只吃「概念/方案/决策/术语定义」类笔记。
          路径白名单(07-Wiki/concepts/项目研究/工作流方法论 等)
          + 目录黑名单(01-Daily/00-Inbox/templates/04-Archive/06-Raw/test/_wiki …)
          + 标题黑名单(日志/清单/索引/日期型标题 …) + 内容黑名单(纯链接页/表格页/日志页)
② 质量门槛 is_quality_fact()：<20 字拒 / 纯疑问句拒 / 状态汇报型拒 /
          指令与提示词片段拒(以『应该/不要/必须/只提取/禁止』开头，或整句含 ≥2 个
          提示词词形 prompt/schema/JSON/facts/IS=/SU= 等) / 代码块与表格行整行拒 /
          事实需含具体名词或结论动词
③ 限量增量 --limit N(默认 10)；state 记 sha256，已处理即跳过；
          单文件产出上限 concepts<=8 / entities<=10 / relations<=12，超出丢弃并计数上报
④ 抽检    默认 dry-run（不写）；--sample-report 输出「拟写入样本 20 条(含来源 文件:行)」
          到 /tmp/wiki-curator-sample-<date>.md；只有显式 --commit 才真写。
          首次启用了流程：dry-run → 人过样本 → --commit 小批 10 篇 → 复查 stats 增量与 recall

Usage:
    python3 wiki_curator.py                          # 抽检(默认 dry-run)，四道闸门全开，不写
    python3 wiki_curator.py --sample-report          # 抽检 + 落样本报告到 /tmp
    python3 wiki_curator.py --limit 10 --commit      # 小批真写（首启流程第二步）
    python3 wiki_curator.py --dir ~/mc/小雪          # 换目录抽检
    python3 wiki_curator.py --force --limit 0        # 忽略 state、不限篇数（仅调试用）

禁止：把 cron 直接改成每日全量；绕过抽检直接大跑。
"""

import argparse
import hashlib
import json
import os
import re
import sys
from datetime import datetime
from pathlib import Path

try:
    import requests
except ImportError:
    print("ERROR: 'requests' library is required. Install with: pip install requests")
    sys.exit(1)

try:
    import yaml
except ImportError:
    print("WARNING: 'yaml' library not available; LLM mode will use env var fallback")
    yaml = None

# ---------------------------------------------------------------------------
# 配置
# ---------------------------------------------------------------------------
ZHIYI_API = "http://localhost:7821"
ZHIYI_KEY = "zhiyi-dev-key-2026"
ZHIYI_AGENT_ID = "hermes-main"   # 与主库一致，recall 才取得到
STATE_FILE = os.path.expanduser("~/.hermes/wiki_curator_state.json")

# LLM 配置
# 本地 llama-server 优先（2026-08-29），Agnes 兜底，NewAPI 最后
AGNES_API = "http://127.0.0.1:3000/v1/chat/completions"   # 统一走 newapi（变量名保留兼容）
AGNES_KEY = ""
_env_path = os.path.expanduser("~/.hermes/.env")
if os.path.isfile(_env_path):
    for _l in open(_env_path, encoding="utf-8"):
        if _l.startswith("AGNES_API_KEY=") and not _l.startswith("#"):
            AGNES_KEY = _l.strip().split("=", 1)[1]
            break
# 2026-09-07：保持云模型(agnes)——深度知识提取(concepts/entities/relations)质量敏感，
# 4B(理解力弱)会产出低质量 JSON 污染知识库。4B 仅用于简单文本/有自动校验的任务。
LLM_API = AGNES_API if AGNES_KEY else "http://127.0.0.1:3000/v1/chat/completions"
LLM_MODEL = "agnes-2.5-flash" if AGNES_KEY else "nvidia/nemotron-3-super-120b-a12b"

# 扫描时排除的目录名称（大小写不敏感）
EXCLUDE_DIRS = {
    "__pycache__", ".git", "node_modules", ".obsidian", ".trash",
    "backups", ".gitlab", ".github", ".vscode", ".idea",
    "venv", ".venv", "env", ".env",
    # 2026-09-11: 凭证/隐私目录，绝不蒸馏进织忆
    "claw",           # 牧尘/claw/ —— key.md 等凭证库
    "key",            # 任何名为 key 的目录
    "凭据", "密钥",
}

# 最小文件长度（字符数）—— 太短的文件没有足够知识量
MIN_CHARS = 500

# ===========================================================================
# 闸门① 选择性 —— 路径白名单 + 目录/标题/内容黑名单
# ===========================================================================

# 路径白名单：相对路径（小写、'/' 分隔）命中任一片段才允许进入。
# 语义 = 「概念 / 方案 / 决策 / 术语定义 / 项目研究 / 工具研究」类知识笔记。
PATH_WHITELIST = (
    "07-wiki",              # 知识库主区（其下再被目录黑名单细分）
    "concepts", "项目研究", "工作流方法论", "references", "参考项目",
    "ai-agent", "ai工具研究", "探索", "流程方法论",
    "obsidian-guide", "opencode配置", "pkm",
    "06-工具与资源", "tools",
)

# 目录黑名单：相对路径**任一层目录名**命中即整支跳过。
PATH_BLACKLIST = {
    "01-daily", "00-inbox", "templates", "template",
    "04-archive", "06-raw", "test", "tests", "test-llm", "test-persist",
    "attachments", "assets", "_wiki", "_trash",
    "碎片信息", "工作记录", "xiaoxuedate", "文学", "旅游",
    "家庭相关", "个人资料", "剪辑学习", "浏览器", "招商工作",
    "ai生成", "内容草稿",
}

# 标题黑名单：文件名（去扩展名）命中即跳过
BAD_TITLE_RE = re.compile(
    r"(日志|日报|周报|月报|清单|待办|计划表|模板|索引|目录"
    r"|readme|changelog|home|index|todo|inbox|toc)",
    re.IGNORECASE,
)
DATE_TITLE_RE = re.compile(r"^\d{4}[-年]\d{1,2}([-月]\d{1,2})?日?$")


def path_stats(rel_path: str):
    """返回 (是否通过, 原因)。闸门① 的路径部分。"""
    low = rel_path.replace(os.sep, "/").lower()
    parts = [p for p in low.split("/") if p]
    # 目录黑名单（只看目录层，文件名不算）
    for p in parts[:-1]:
        if p in PATH_BLACKLIST:
            return False, f"dir_blacklist:{p}"
    stem = parts[-1]
    if stem.endswith(".md"):
        stem = stem[:-3]
    if stem in PATH_BLACKLIST:
        return False, "file_blacklist"
    # 白名单
    if not any(w in low for w in PATH_WHITELIST):
        return False, "not_in_whitelist"
    # 标题黑名单
    if BAD_TITLE_RE.search(stem) or DATE_TITLE_RE.match(stem):
        return False, "title_blacklist"
    return True, "ok"


_LINK_LINE_RE = re.compile(r"^\s*(?:[-*>]\s*)?(?:\[[^\]]*\]\([^)]*\)|!\[|https?://|\d+\.\s*\[)")
_LOG_LINE_RE = re.compile(r"^\s*(?:[-*]\s*)?(?:\[[ xX]?\]\s*)?\d{4}[-/年]\d{1,2}")


def content_blacklisted(content: str) -> str:
    """闸门① 的内容部分：纯链接页 / 表格页 / 日志页 → 返回原因，否则空串。"""
    lines = [l.strip() for l in content.splitlines() if l.strip()]
    if not lines:
        return "empty"
    n = len(lines)
    links = sum(1 for l in lines if _LINK_LINE_RE.match(l))
    if links / n > 0.6:
        return "link_page"
    tables = sum(1 for l in lines if l.startswith("|"))
    if tables / n > 0.5:
        return "table_page"
    logs = sum(1 for l in lines if _LOG_LINE_RE.match(l))
    if logs / n > 0.4:
        return "log_page"
    return ""


# ===========================================================================
# 闸门② 质量门槛 —— is_quality_fact()
# ===========================================================================

MIN_FACT_CHARS = 20          # <20 字拒绝
STATUS_NARROW_CHARS = 60     # 状态汇报型只在「短句」上拒，避免误杀长段落

# 提示词词形（prompt 片段特征）。原规范 5 个，加固后补 JSON 字段名（带引号）——
# 设计文档里的 schema 示例靠字段名命中。
PROMPT_FORMS = (
    "prompt", "schema", "json", "facts", "is=", "su=",
    'name"', 'summary"', 'attributes"', 'relation"', 'target"', 'source"',
)
IMPERATIVE_HEADS = ("应该", "应当", "不要", "必须", "只提取", "禁止", "严禁")
STATUS_NOISE = (
    "健康检查", "状态同步", "状态汇报", "心跳", "heartbeat", "桥接",
    "测试成功", "验证完成", "运行正常", "已完成", "已重启", "已恢复",
    "reflection", "daemon tick", "check ok",
)
QUESTION_TAIL_RE = re.compile(r"[?？]\s*$")
QUESTION_HEAD_RE = re.compile(
    r"^\s*(是否|为什么|为何|怎么|如何|什么|哪些|哪一|何时|谁|能不能|可不可以|要不要|有没有)"
)
TABLE_ROW_RE = re.compile(r"^\s*\|.*\|\s*$")
CODE_FENCE_RE = re.compile(r"```|~~~")
# 「具体名词」近似：≥2 连续汉字 或 ≥3 连续字母数字
NOUN_SUBSTANCE_RE = re.compile(r"[\u4e00-\u9fff]{2,}|[A-Za-z0-9_]{3,}")
QUOTED_TERM_RE = re.compile(r"[\"“”『』「」'].+[\"“”『』「」']")
CONCLUSION_VERBS = (
    "决定", "采用", "选择", "确认", "结论", "定义", "方案", "架构",
    "实现", "负责", "属于", "位于", "基于", "依赖", "使用", "命名",
    "部署", "计划", "要求", "用于", "指的是", "是",
)

_REASON_LABEL = {
    "empty": "空内容",
    "code_block": "代码块",
    "code_or_table": "代码/表格行",
    "heading_capture": "标题抓取",
    "heading_lead": "标题引导句",
    "table_row": "表格行整行",
    "too_short": f"<{MIN_FACT_CHARS}字",
    "question": "纯疑问句",
    "imperative": "指令句开头",
    "prompt_fragment": "提示词片段",
    "status_report": "状态汇报型",
    "no_substance": "无具体名词",
}


def is_quality_fact(text: str):
    """闸门②：判断一段待写入文本是否够格进织忆。

    返回 (ok: bool, reason: str)。reason 恒为英文 key（用于计数），
    label 见 _REASON_LABEL。
    """
    if not text:
        return False, "empty"
    t = text.strip()
    # 去掉 markdown 前缀标记（标题 # / 引用 > / 列表 - *）
    probe = re.sub(r"^[>\-*\s#]+", "", t).strip()
    if not probe:
        return False, "empty"
    # 注：不做「以 # 开头即拒」——闸门自己会拼 "## 名\n摘要" 前缀，那样会误杀全部
    # 只拒「标题 + 半句引导」（真正的标题抓取形态：短且以冒号收尾）
    if probe.rstrip().endswith(("：", ":")) and len(probe) < 60:
        return False, "heading_lead"
    # 代码块 / 表格行整行
    if CODE_FENCE_RE.search(probe):
        return False, "code_block"
    if "\n" in probe:
        for ln in probe.splitlines():
            if TABLE_ROW_RE.match(ln) or ln.strip().startswith("|"):
                return False, "code_or_table"
    else:
        if TABLE_ROW_RE.match(probe) or probe.startswith("|"):
            return False, "table_row"

    flat = re.sub(r"\s+", " ", probe).strip()
    # 指令 / 提示词片段（headline gate：把"关于提取的说明书"当"提取结果"）
    if probe.startswith(IMPERATIVE_HEADS):
        return False, "imperative"
    low = flat.lower()
    hits = sum(1 for w in PROMPT_FORMS if w in low)
    if hits >= 2 or (hits >= 1 and QUOTED_TERM_RE.search(flat)):
        return False, "prompt_fragment"
    # 纯疑问句
    if QUESTION_TAIL_RE.search(flat) or QUESTION_HEAD_RE.match(flat):
        return False, "question"
    # 状态汇报型
    if len(flat) < STATUS_NARROW_CHARS and any(w in low for w in STATUS_NOISE):
        return False, "status_report"
    # 长度下限
    if len(flat) < MIN_FACT_CHARS:
        return False, "too_short"
    # 事实需含具体名词或结论动词
    if not NOUN_SUBSTANCE_RE.search(flat) and not any(v in flat for v in CONCLUSION_VERBS):
        return False, "no_substance"
    return True, "ok"


# ===========================================================================
# 闸门③ 限量 + 增量
# ===========================================================================

DEFAULT_LIMIT = 10
# 单文件产出上限。可用 --max-concepts/--max-entities/--max-relations 覆盖。
# 注意：织忆 commit 之后有异步蒸馏（AutoDistillTrigger）会把每条记忆再展开成
# 1~3 个图谱节点，所以「图谱节点增量 ≈ 2.5 × 上游条目数」——要压图谱预算就得压这里。
PER_FILE_LIMITS = {"concepts": 8, "entities": 10, "relations": 12}

# ===========================================================================
# 闸门④ 抽检
# ===========================================================================

SAMPLE_SIZE = 20
PER_FILE_SAMPLE = 3       # 每个文件最多进样本池几条（保证样本跨文件分布）
SAMPLES = []              # [(kind, text, rel_path, line)] —— 最终 20 条（跨文件轮询）
_FILE_SAMPLES = {}        # rel_path -> [(kind, text, rel_path, line)]
_FILE_ORDER = []          # 样本池文件顺序
REJECTED = {}             # reason -> count
DROPPED = {}              # kind -> count（超单文件上限被丢）


def record_sample(kind: str, text: str, source: str, line: int):
    """按文件缓存样本（单文件最多 PER_FILE_SAMPLE 条），最后跨文件轮询取 20 条。"""
    bucket = _FILE_SAMPLES.setdefault(source, [])
    if len(bucket) >= PER_FILE_SAMPLE:
        return
    if not bucket:
        _FILE_ORDER.append(source)
    bucket.append((kind, text, source, line))


def finalize_samples():
    """跨文件轮询挑 SAMPLE_SIZE 条，避免样本全来自同一个文件。"""
    global SAMPLES
    picked = []
    idx = 0
    pools = [_FILE_SAMPLES[p] for p in _FILE_ORDER if _FILE_SAMPLES.get(p)]
    while len(picked) < SAMPLE_SIZE and pools:
        progressed = False
        for pool in pools:
            if idx < len(pool) and len(picked) < SAMPLE_SIZE:
                picked.append(pool[idx])
                progressed = True
        idx += 1
        if not progressed:
            break
    SAMPLES = picked
    return SAMPLES


def sample_report_path() -> str:
    return f"/tmp/wiki-curator-sample-{datetime.now().strftime('%Y%m%d')}.md"


def write_sample_report(path: str, stats: dict, meta: dict) -> str:
    lines = [
        f"# Wiki Curator 拟写入样本 — {datetime.now().strftime('%Y-%m-%d %H:%M')}",
        "",
        f"- 模式: {meta.get('mode')}",
        f"- 扫描目录: {meta.get('scan_dir')}",
        f"- 本批处理文件: {meta.get('files')}（本轮上限 {meta.get('limit')}）",
        f"- 闸门① 选择性: 扫描 {meta.get('scanned')} 篇 → 命中 {meta.get('selected')} 篇，"
        f"跳过 {meta.get('skipped')} 篇",
        f"- 闸门② 质量门槛: 拒绝 {meta.get('rejected_total')} 条"
        f"（{meta.get('rejected_desc') or '无'}）",
        f"- 闸门③ 单文件上限: concepts<={PER_FILE_LIMITS['concepts']} / "
        f"entities<={PER_FILE_LIMITS['entities']} / relations<={PER_FILE_LIMITS['relations']}，"
        f"丢弃 {meta.get('dropped_total')} 条",
        f"- 拟写入: 概念 {stats['concepts']} + 实体 {stats['entities']} + 关系 {stats['relations']}",
        "",
        f"## 样本（前 {len(SAMPLES)} 条，含来源 文件:行）",
        "",
    ]
    for i, (kind, text, src, line) in enumerate(SAMPLES, 1):
        flat = re.sub(r"\s+", " ", text).strip()
        lines.append(f"{i}. [{kind}] {flat[:160]}")
        lines.append(f"   - 来源: {src}:{line}")
    lines.append("")
    with open(path, "w", encoding="utf-8") as f:
        f.write("\n".join(lines))
    return path


# ---------------------------------------------------------------------------
# 工具函数
# ---------------------------------------------------------------------------

def compute_sha256(content: str) -> str:
    """计算字符串的 SHA-256 摘要。"""
    return hashlib.sha256(content.encode("utf-8")).hexdigest()


def load_state() -> dict:
    """加载已处理文件的哈希状态。"""
    if os.path.isfile(STATE_FILE):
        try:
            with open(STATE_FILE, "r", encoding="utf-8") as f:
                return json.load(f)
        except (json.JSONDecodeError, OSError):
            print("  [WARN] 状态文件损坏，重置为空。")
    return {}


def save_state(state: dict):
    """保存处理状态到文件。"""
    os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
    with open(STATE_FILE, "w", encoding="utf-8") as f:
        json.dump(state, f, ensure_ascii=False, indent=2)


def should_exclude_dir(dirname: str) -> bool:
    """检查目录名是否在排除列表中。"""
    return dirname.lower() in EXCLUDE_DIRS


ONLY_SLOTS = []      # 闸门① 白名单：非空的只吃这些槽位目录（2026-09-11 加）


def scan_md_files(scan_dir: str, force: bool, state: dict, quiet: bool = True) -> tuple:
    """递归扫描 .md 文件，返回 (candidates, gate_stats)。

    candidates: [(相对路径, 绝对路径, 内容, 文件哈希)]
    gate_stats: 闸门① 的计数（scanned / skipped / skip_reasons）
    """
    scan_path = Path(scan_dir).expanduser().resolve()
    if not scan_path.is_dir():
        print(f"  [WARN] 目录不存在: {scan_path}")
        return [], {"scanned": 0, "selected": 0, "skipped": 0, "reasons": {}}

    candidates = []
    gate = {"scanned": 0, "selected": 0, "skipped": 0, "reasons": {}}

    def _bump(reason):
        gate["reasons"][reason] = gate["reasons"].get(reason, 0) + 1

    for root_str, dirs, files in os.walk(str(scan_path)):
        # 过滤排除目录（原地修改 dirs 避免继续深入）
        dirs[:] = [d for d in dirs if not should_exclude_dir(d)]

        for fn in files:
            if not fn.endswith(".md") or fn.startswith("_"):
                continue
            gate["scanned"] += 1

            abs_path = Path(root_str) / fn
            rel_path = abs_path.relative_to(scan_path)

            # 闸门① 白名单：只吃知识槽位（定义规范/方案设计），跳过日志/素材等过程记录
            if ONLY_SLOTS and not any(s in str(rel_path) for s in ONLY_SLOTS):
                _bump("未命中知识槽位")
                gate["skipped"] += 1
                continue

            # 闸门① 路径选择性（在黑名单里的直接跳过，连读都不读）
            # 2026-09-11 修：若已用 --only-slots 指定知识槽位，则槽位命中即放行 ——
            # 槽位（定义规范/方案设计/交付定稿…）本身就是"知识型"判据，
            # 再套 PATH_WHITELIST（那串只覆盖 05-AI-AGENT 旧区名）会把
            # 01-织忆/06-方法论/04-OBSIDIAN/小米笔记本运维 全部误挡 → 覆盖不足。
            ok, reason = path_stats(str(rel_path))
            if not ok and ONLY_SLOTS and any(s in str(rel_path) for s in ONLY_SLOTS):
                ok, reason = True, "slot_override"
            if not ok:
                gate["skipped"] += 1
                _bump(reason)
                continue

            try:
                content = abs_path.read_text(encoding="utf-8")
            except (OSError, UnicodeDecodeError) as e:
                print(f"  [WARN] 读取失败 {abs_path}: {e}")
                gate["skipped"] += 1
                _bump("read_error")
                continue

            if len(content) < MIN_CHARS:
                gate["skipped"] += 1
                _bump("too_short_file")
                continue

            # 闸门① 内容选择性
            creason = content_blacklisted(content)
            if creason:
                gate["skipped"] += 1
                _bump(f"content_{creason}")
                continue

            file_hash = compute_sha256(content)
            key = str(rel_path)

            # 闸门③ 增量：未变更则跳过
            if not force and state.get(key) == file_hash:
                gate["skipped"] += 1
                _bump("unchanged")
                continue

            gate["selected"] += 1
            candidates.append((key, str(abs_path), content, file_hash))

    return candidates, gate


# ---------------------------------------------------------------------------
# 知识提取（启发式 / 基于关键词）
# ---------------------------------------------------------------------------

def _line_of(content: str, pos: int) -> int:
    return content.count("\n", 0, pos) + 1


def _heuristic_extract(content: str) -> dict:
    """
    从 markdown 内容中提取知识点（启发式方法）。
    返回结构：
        {"concepts": [{"name","summary","line"}],
         "entities": [{"name","attributes","line"}],
         "relations": []}
    """
    concepts = []
    entities = []

    # 1. 提取 heading 作为概念名称
    heading_pattern = re.compile(r"^#{1,6}\s+(.+)$", re.MULTILINE)
    for match in heading_pattern.finditer(content):
        heading_text = match.group(1).strip()
        if not heading_text or len(heading_text) < 2:
            continue
        # 收集该标题下的文本（直到下一个标题或文件末尾）
        start_pos = match.end()
        next_heading = heading_pattern.search(content, start_pos)
        if next_heading:
            section_content = content[start_pos:next_heading.start()].strip()
        else:
            section_content = content[start_pos:].strip()

        summary = ""
        for line in section_content.split("\n"):
            line = line.strip()
            if not line:
                continue
            if line.startswith("#") or line.startswith("-"):
                continue
            # Skip metadata lines: blockquotes, HTML anchors, specific markers
            if line.startswith("> ") or line.startswith("<a name=") or line.startswith("<a id="):
                continue
            if line.strip() in ["**现状**：", "**状态**：", "**注意**：", "**提示**：", "**摘要**："]:
                continue
            # If we get here, this looks like actual content
            summary = line[:200]
            break

        concepts.append({
            "name": heading_text,
            "summary": summary,
            "line": _line_of(content, match.start()),
        })

    # 2. 提取 **粗体** 关键词作为实体
    bold_pattern = re.compile(r"\*\*(.+?)\*\*")
    seen_bolds = set()
    for match in bold_pattern.finditer(content):
        bold_text = match.group(1).strip()
        if not bold_text or len(bold_text) > 80:
            continue
        if bold_text.lower() in seen_bolds:
            continue
        seen_bolds.add(bold_text.lower())

        start = max(0, match.start() - 50)
        end = min(len(content), match.end() + 50)
        context = content[start:end].replace("\n", " ").strip()
        context = re.sub(r"\s+", " ", context)

        entities.append({
            "name": bold_text,
            "attributes": context[:200],
            "line": _line_of(content, match.start()),
        })

    # 3. 提取列表项中的重要短语
    list_pattern = re.compile(r"^[\s]*[-*]\s+(.+)$", re.MULTILINE)
    seen_list_items = set()
    for match in list_pattern.finditer(content):
        item_text = match.group(1).strip()
        if not item_text or item_text.startswith("[") or item_text.startswith("!"):
            continue
        if len(item_text) < 4:
            continue
        if item_text.lower() in seen_list_items:
            continue
        seen_list_items.add(item_text.lower())

        if "：" in item_text or ":" in item_text:
            parts = re.split(r"[：:]", item_text, maxsplit=1)
            name = parts[0].strip()
            desc = parts[1].strip() if len(parts) > 1 else ""
        else:
            bm = re.search(r"\*\*(.+?)\*\*", item_text)
            if bm:
                name = bm.group(1).strip()
                desc = re.sub(r"\*\*(.+?)\*\*", r"\1", item_text)
            else:
                name = item_text[:60]
                desc = item_text

        if len(name) < 2:
            continue

        entities.append({
            "name": name,
            "attributes": desc[:200],
            "line": _line_of(content, match.start()),
        })

    return {"concepts": concepts, "entities": entities}


# ---------------------------------------------------------------------------
# LLM 知识提取
# ---------------------------------------------------------------------------

def _get_llm_key() -> str:
    """从 config.yaml 读取 NewAPI key"""
    if yaml is not None:
        try:
            cfg_path = os.path.expanduser("~/.hermes/config.yaml")
            with open(cfg_path) as f:
                cfg = yaml.safe_load(f)
            raw = cfg.get("providers", {}).get("newapi-local", {}).get("api_key", "")
            return raw
        except Exception:
            pass
    return AGNES_KEY or os.environ.get("NEWAPI_API_KEY", "")


def _extract_with_llm(content: str, filepath: str) -> dict | None:
    """调用 LLM 提取结构化知识（key 必须与端点匹配）"""
    # 🔴 2026-09-18 修（脚本层直连清理）：LLM_API 已统一到网关，上游 agnes key 打到
    #    网关会 401（实测 Invalid token）→ key 与端点必须配对，统一用 config.yaml 的网关 key。
    #    2026-09-11 旧注释保留：端点=agnes 直连时不能用 NewAPI 的 key（曾导致"无效的令牌"全体回退）
    llm_key = _get_llm_key()
    if not llm_key:
        print("  ⚠️ No LLM API key found (check config.yaml or NEWAPI_API_KEY env)")
        return None

    prompt = f'''Extract concepts, entities, and relations from this document.
Return ONLY valid JSON:
{{"concepts":[{{"name":"...","summary":"..."}}],"entities":[{{"name":"...","attributes":{{}}}}],"relations":[{{"source":"...","relation":"uses|contains|depends_on|part_of|implements","target":"..."}}]}}

Document:
{content[:2000]}
'''
    try:
        resp = requests.post(LLM_API,
            headers={"Authorization": f"Bearer {llm_key}", "Content-Type": "application/json"},
            json={
                "model": LLM_MODEL,
                "messages": [
                    {"role": "system", "content": "You are a knowledge extraction assistant. Always respond with valid JSON only."},
                    {"role": "user", "content": prompt}
                ],
                "temperature": 0.1,
                "max_tokens": 3000,   # Agnes 是推理模型，小额度会被 reasoning 吃光致 content 空（8-17 老坑）
            },
            timeout=120)   # 2026-09-12 从 30 提到 120：负载高时 30s 会丢写（实测 load 23 时失败）
        data = resp.json()
        if "error" in data and data["error"].get("message"):
            print(f"  ⚠️ LLM API error: {data['error']['message'][:60]}")
            return None
        choices = data.get("choices", [])
        if not choices:
            print("  ⚠️ LLM returned empty choices (API/model may be unavailable)")
            return None
        msg = choices[0].get("message", {})
        text = msg.get("content", "") or ""
        if not text.strip():
            finish = choices[0].get("finish_reason", "")
            print(f"  ⚠️ LLM returned empty content (finish={finish})")
            return None
        json_match = re.search(r'\{[\s\S]*\}', text)
        if json_match:
            return json.loads(json_match.group())
    except Exception as e:
        print(f"  ⚠️ LLM extraction failed: {e}")
    return None


# ---------------------------------------------------------------------------
# 织忆 API 交互
# ---------------------------------------------------------------------------

def _post_with_retry(url, headers, payload, tries=4, tag="写入"):
    """带退避重试的 POST。429 时读 retry_after 等待；其它错误短退避。"""
    import time as _t
    for i in range(tries):
        try:
            resp = requests.post(url, json=payload, headers=headers, timeout=120)  # 同上，防高负载丢写
            if resp.status_code in (200, 201):
                return True, resp
            if resp.status_code == 429:
                wait = 1.0
                try:
                    wait = float(resp.json().get("retry_after", 1)) 
                except Exception:
                    pass
                wait = max(wait, 1.0) * (2 ** i)          # 指数退避
                print(f"    [429] {tag} 限流，{wait:.1f}s 后重试 ({i+1}/{tries})")
                _t.sleep(wait)
                continue
            print(f"    [FAIL] {tag} HTTP {resp.status_code}: {resp.text[:160]}")
            return False, resp
        except requests.exceptions.RequestException as e:
            if i == tries - 1:
                print(f"    [FAIL] {tag} 请求失败(已重试{tries}次): {e}")
                return False, None
            _t.sleep(1.0 * (2 ** i))
    print(f"    [FAIL] {tag} 重试耗尽")
    return False, None


def commit_memory(content: str, category: str, metadata: dict,
                  dry_run: bool = False) -> bool:
    """写入一条记忆到织忆。返回 True 表示成功。

    ⚠️ 2026-09-11 修：原 agent_id="wiki-curator" → 记忆库里**查不到该命名空间**
    （namespace=wiki-curator 实测 0 条），recall 永远召不回笔记知识。
    改为与主库一致的 agent_id / namespace，让 recall 能取到。
    """
    if dry_run:
        return True
    metadata = dict(metadata or {})
    metadata.setdefault("source", "wiki-curator")
    metadata.setdefault("vault", "mc")
    payload = {
        "agent_id": ZHIYI_AGENT_ID,
        "namespace": ZHIYI_AGENT_ID,
        "content": content,
        "category": category,
        "metadata": metadata,
    }
    ok, _ = _post_with_retry(f"{ZHIYI_API}/api/v1/commit",
                             {"X-API-Key": ZHIYI_KEY, "Content-Type": "application/json"},
                             payload, tag="记忆写入")
    return ok


def commit_graph_edge(from_node: str, to_node: str, relation: str,
                      dry_run: bool = False) -> bool:
    """写入一条关系到织忆图谱。返回 True 表示成功。"""
    if dry_run:
        return True

    url = f"{ZHIYI_API}/api/v1/graph/edge"
    headers = {
        "X-API-Key": ZHIYI_KEY,
        "Content-Type": "application/json",
    }
    payload = {
        "from": from_node,
        "to": to_node,
        "relation": relation,
        "namespace": "wiki",
    }
    ok, _ = _post_with_retry(url, headers, payload, tag="关系写入")
    return ok


# ---------------------------------------------------------------------------
# 主流程
# ---------------------------------------------------------------------------

def _gate_items(items, kind, build_text):
    """对一组候选跑闸门②（质量门槛）+ 闸门③（单文件上限）。

    build_text(item) -> 待写入文本
    返回 (kept, dropped_count)
    """
    kept = []
    for it in items:
        text = build_text(it)
        ok, reason = is_quality_fact(text)
        if not ok:
            REJECTED[reason] = REJECTED.get(reason, 0) + 1
            continue
        kept.append(it)
    cap = PER_FILE_LIMITS.get(kind, 10 ** 9)
    if len(kept) > cap:
        DROPPED[kind] = DROPPED.get(kind, 0) + (len(kept) - cap)
        kept = kept[:cap]
    return kept


def process_file(rel_path: str, abs_path: str, content: str,
                 dry_run: bool = False, use_llm: bool = False) -> dict:
    """处理单个文件：提取知识点 → 闸门②③ → 写入（或抽检）。"""
    stats = {"concepts": 0, "entities": 0, "relations": 0}

    # 提取知识
    if use_llm:
        result = _extract_with_llm(content, abs_path)
        if result:
            concepts = result.get("concepts", [])
            entities = result.get("entities", [])
            relations = result.get("relations", [])
            print(f"  📄 {rel_path}  [LLM] 原始 {len(concepts)}概念/{len(entities)}实体/{len(relations)}关系")
        else:
            print(f"  📄 {rel_path}  [LLM 失败 → heuristic]")
            knowledge = _heuristic_extract(content)
            concepts = knowledge.get("concepts", [])
            entities = knowledge.get("entities", [])
            relations = []
    else:
        knowledge = _heuristic_extract(content)
        concepts = knowledge.get("concepts", [])
        entities = knowledge.get("entities", [])
        relations = []

    for c in concepts:
        c.setdefault("line", 0)
    for e in entities:
        e.setdefault("line", 0)

    # ---- 闸门② 质量门槛 + 闸门③ 单文件上限 ----
    concepts = _gate_items(
        concepts, "concepts",
        lambda c: f"{c['name']}\n{c.get('summary', '')}",
    )
    entities = _gate_items(
        entities, "entities",
        lambda e: f"{e['name']}\n{_attrs_str(e.get('attributes'))}",
    )

    # 写入概念
    for conc in concepts:
        content_line = f"## {conc['name']}"
        if conc.get("summary"):
            content_line += f"\n{conc['summary']}"
        metadata = {"source": abs_path, "concept_type": "concept"}
        if commit_memory(content_line, "wiki", metadata, dry_run=dry_run):
            stats["concepts"] += 1
            record_sample("concept", content_line, rel_path, conc.get("line", 0))

    # 写入实体
    for ent in entities:
        content_line = f"### {ent['name']}"
        attrs_str = _attrs_str(ent.get("attributes"))
        if attrs_str:
            content_line += f"\n{attrs_str}"
        metadata = {"source": abs_path, "concept_type": "entity"}
        if commit_memory(content_line, "wiki", metadata, dry_run=dry_run):
            stats["entities"] += 1
            record_sample("entity", content_line, rel_path, ent.get("line", 0))

    # 写入关系（闸门③ 上限）
    if relations:
        cap = PER_FILE_LIMITS["relations"]
        if len(relations) > cap:
            DROPPED["relations"] = DROPPED.get("relations", 0) + (len(relations) - cap)
            relations = relations[:cap]
        for rel in relations:
            source_node = rel.get("source", "")
            target_node = rel.get("target", "")
            relation_type = rel.get("relation", "RELATED_TO").upper()
            if source_node and target_node:
                if commit_graph_edge(source_node, target_node, relation_type, dry_run=dry_run):
                    stats["relations"] += 1
    elif concepts and entities:
        # 补概念-实体关系（同样受单文件上限约束）
        primary = concepts[0]["name"]
        pairs = [(primary, e["name"]) for e in entities][:PER_FILE_LIMITS["relations"]]
        for src, tgt in pairs:
            ok, _ = is_quality_fact(f"{src} {tgt}")
            if not ok:
                continue
            if commit_graph_edge(src, tgt, "RELATED_TO", dry_run=dry_run):
                stats["relations"] += 1

    print(f"  📄 {rel_path}  → 概念 {stats['concepts']} / 实体 {stats['entities']} / 关系 {stats['relations']}")
    return stats


def _attrs_str(attrs) -> str:
    if not attrs:
        return ""
    if isinstance(attrs, dict):
        return json.dumps(attrs, ensure_ascii=False)
    return str(attrs)


def main():
    parser = argparse.ArgumentParser(
        description="Wiki Curator — 自动知识策展管线 (P5)",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""四种模式（四道闸门全开，缺一不开）:
  ① 抽检模式(默认)   python3 wiki_curator.py
      默认 dry-run，什么都不写；只打印每篇的拟写入计数。
  ② 样本报告模式     python3 wiki_curator.py --sample-report
      在前者基础上，把「拟写入样本 20 条（含来源 文件:行）」
      落到 /tmp/wiki-curator-sample-<date>.md，供人过目。
  ③ 真写模式         python3 wiki_curator.py --limit 10 --commit
      只有显式 --commit 才写织忆；--limit 限制本轮篇数（默认 10）。
      首启流程：dry-run → 人过样本 → --commit 小批 10 篇 → 复查 stats 增量与 recall。
  ④ 调试模式         python3 wiki_curator.py --force --limit 0 --dry-run
      忽略 state 已处理记录、不限篇数，仅用于本地调试。

闸门：① 选择性(路径白名单+目录/标题/内容黑名单) ② 质量门槛(is_quality_fact)
      ③ 限量增量(--limit / sha256 增量 / 单文件上限 --max-concepts/--max-entities/--max-relations)
      ④ 抽检(默认 dry-run + --sample-report，只有 --commit 才写)

图谱预算提示：织忆 commit 后还有异步蒸馏会把每条记忆展开成 1~3 个图谱节点，
所以「图谱节点增量 ≈ 2.5 × 上游条目数」。要压图谱增量就调 --max-* 三项。
""",
    )
    parser.add_argument(
        "--dry-run", action="store_true",
        help="抽检模式：不写入织忆（默认行为，显式传入可覆盖 --commit）"
    )
    parser.add_argument(
        "--commit", action="store_true",
        help="真写模式：真正写入织忆（不加此项一律不写）"
    )
    parser.add_argument(
        "--sample-report", action="store_true",
        help=f"输出「拟写入样本 {SAMPLE_SIZE} 条（含来源 文件:行）」到 /tmp/wiki-curator-sample-<date>.md"
    )
    parser.add_argument(
        "--limit", type=int, default=DEFAULT_LIMIT,
        help=f"本轮最多处理多少篇（默认 {DEFAULT_LIMIT}；<=0 表示不限，仅调试）"
    )
    parser.add_argument(
        "--max-concepts", type=int, default=PER_FILE_LIMITS["concepts"],
        help=f"单文件概念产出上限（默认 {PER_FILE_LIMITS['concepts']}，超出丢弃并计数）"
    )
    parser.add_argument(
        "--max-entities", type=int, default=PER_FILE_LIMITS["entities"],
        help=f"单文件实体产出上限（默认 {PER_FILE_LIMITS['entities']}，超出丢弃并计数）"
    )
    parser.add_argument(
        "--max-relations", type=int, default=PER_FILE_LIMITS["relations"],
        help=f"单文件关系产出上限（默认 {PER_FILE_LIMITS['relations']}，超出丢弃并计数）"
    )
    parser.add_argument(
        "--dir", type=str, default="~/mc/小怡/",
        help="扫描目录 (默认: ~/mc/小怡/)"
    )
    parser.add_argument(
        "--only-slots", type=str, default="01-定义规范,02-方案设计",
        help="闸门① 白名单：只吃这些槽位目录（逗号分隔；传空字符串=不过滤）"
    )
    parser.add_argument(
        "--force", action="store_true",
        help="忽略状态文件，重新处理所有文件"
    )
    parser.add_argument(
        "--llm", action="store_true",
        help="Use LLM for concept/entity extraction (default: heuristic)"
    )
    args = parser.parse_args()
    global ONLY_SLOTS
    ONLY_SLOTS = [s.strip() for s in args.only_slots.split(",") if s.strip()]

    # 闸门③ 单文件上限：CLI 可覆盖
    PER_FILE_LIMITS["concepts"] = max(0, args.max_concepts)
    PER_FILE_LIMITS["entities"] = max(0, args.max_entities)
    PER_FILE_LIMITS["relations"] = max(0, args.max_relations)

    # 闸门④：只有显式 --commit 且未显式 --dry-run 才真写
    do_write = args.commit and not args.dry_run
    mode = "COMMIT(真写)" if do_write else "DRY-RUN(抽检，不写)"

    print("=" * 60)
    print("  织忆 Wiki Curator — 知识策展管线 (四道闸门)")
    print("=" * 60)

    scan_dir = os.path.expanduser(args.dir)
    print(f"\n扫描目录: {scan_dir}")
    print(f"模式: {mode}")
    if args.limit > 0:
        print(f"闸门③ 本轮上限: {args.limit} 篇")
    else:
        print("闸门③ 本轮上限: 无限制（调试）")
    if args.force:
        print("闸门③ 增量: FORCE（忽略已有状态）")

    state = {} if args.force else load_state()
    print(f"状态文件: {STATE_FILE}")
    print(f"已处理文件(state): {len(state)}")

    # 闸门① 选择性 + 闸门③ 增量
    candidates, gate = scan_md_files(scan_dir, args.force, state)
    print(f"\n闸门① 选择性: 扫描 {gate['scanned']} 篇 → 命中 {gate['selected']} 篇，"
          f"跳过 {gate['skipped']} 篇")
    if gate["reasons"]:
        top = sorted(gate["reasons"].items(), key=lambda kv: -kv[1])[:8]
        print("  跳过原因: " + ", ".join(f"{k}={v}" for k, v in top))

    selected_total = len(candidates)
    if args.limit > 0:
        candidates = candidates[:args.limit]
        if selected_total > len(candidates):
            print(f"闸门③ 限量: 本轮处理 {len(candidates)} / 待处理 {selected_total} 篇"
                  f"（其余下轮按增量继续）")
    print(f"\n本批处理文件: {len(candidates)}")

    total_stats = {"concepts": 0, "entities": 0, "relations": 0}
    new_state = dict(state)
    processed = 0

    for rel_path, abs_path, content, file_hash in candidates:
        stats = process_file(rel_path, abs_path, content,
                             dry_run=not do_write, use_llm=args.llm)
        total_stats["concepts"] += stats["concepts"]
        total_stats["entities"] += stats["entities"]
        total_stats["relations"] += stats["relations"]
        processed += 1
        # 闸门③ 增量：只有真写过才记 state（dry-run 不记，避免"抽检即视为已处理"）
        if do_write:
            new_state[rel_path] = file_hash
            # 逐文件落盘：进程被 kill/超时也不丢进度，重跑自动跳过已完成文件
            save_state(new_state)
        print(f"  [进度] {processed}/{len(candidates)} {rel_path}", flush=True)

    if do_write:
        save_state(new_state)
        print(f"\n状态已更新: {len(new_state)} 个文件记录")
    else:
        print("\n[dry-run] state 未变更（抽检不视为已处理）")

    rejected_total = sum(REJECTED.values())
    dropped_total = sum(DROPPED.values())
    rejected_desc = ", ".join(
        f"{_REASON_LABEL.get(k, k)}={v}"
        for k, v in sorted(REJECTED.items(), key=lambda kv: -kv[1])
    )

    # 闸门④ 样本报告
    report_path = None
    if args.sample_report:
        finalize_samples()
        report_path = write_sample_report(sample_report_path(), total_stats, {
            "mode": mode,
            "scan_dir": scan_dir,
            "files": len(candidates),
            "limit": args.limit if args.limit > 0 else "∞",
            "scanned": gate["scanned"],
            "selected": gate["selected"],
            "skipped": gate["skipped"],
            "rejected_total": rejected_total,
            "rejected_desc": rejected_desc,
            "dropped_total": dropped_total,
        })

    print("\n" + "=" * 60)
    print("  📊 处理总结")
    print(f"  模式:         {mode}")
    print(f"  处理文件数:   {len(candidates)}")
    print(f"  概念写入数:   {total_stats['concepts']}")
    print(f"  实体写入数:   {total_stats['entities']}")
    print(f"  关系写入数:   {total_stats['relations']}")
    print(f"  闸门① 跳过:   {gate['skipped']} 篇")
    print(f"  闸门② 质量拒绝: {rejected_total} 条 ({rejected_desc or '无'})")
    print(f"  闸门③ 上限丢弃: {dropped_total} 条 "
          f"({', '.join(f'{k}={v}' for k, v in DROPPED.items()) or '无'})")
    if report_path:
        print(f"  闸门④ 样本报告: {report_path} ({len(SAMPLES)} 条)")
    print("=" * 60)

    print("\nWIKI_CURATOR_OK")
    return 0


if __name__ == "__main__":
    sys.exit(main())