xiaowei-system/scripts/cangjie_distill.py

#!/usr/bin/env python3
"""
仓颉 skill 集成器 — 知识蒸馏引擎
================================
把书/视频/播客里的方法论，蒸馏成可调用的 AI skills。
集成到小唯的学习体系中。

用法:
  cangjie_distill.py distill <source_text> <title>   → 蒸馏文本为 skill
  cangjie_distill.py phase <n> <source_text> <title> → 只跑指定阶段
  cangjie_distill.py verify <skill_dir>             → 压力测试验证
"""

import json, os, re, sys, subprocess
from datetime import datetime
from pathlib import Path

HOME = os.path.expanduser("~")
HERMES = HOME + "/.hermes"
CJ = HERMES + "/cangjie-skill"
OUTPUT = HERMES + "/cangjie-skills"
os.makedirs(OUTPUT, exist_ok=True)


def log(msg):
    print(f"[CANGJIE] {msg}", flush=True)


def shell(cmd, timeout=30):
    try:
        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
        return r.returncode, r.stdout.strip(), r.stderr.strip()
    except subprocess.TimeoutExpired:
        return -1, "", "timeout"


# ===================== RIA-TV++ 流水线 =====================

def stage0_overview(source_text, title):
    """阶段0: 整体内容理解（Adler分析阅读法）"""
    log("Stage 0: 整体内容理解...")
    system_prompt = open(CJ + "/methodology/01-stage0-adler.md").read()

    prompt = f"""请用Adler分析阅读法分析以下内容，输出结构化的 BOOK_OVERVIEW.md：

内容标题: {title}

---
{source_text[:8000]}
---"""
    return call_llm(prompt, system_prompt)


def stage1_extract(source_text, title):
    """阶段1: 并行提取5类方法论单元"""
    log("Stage 1: 并行提取...")

    extractors = {
        "framework": CJ + "/extractors/framework-extractor.md",
        "principle": CJ + "/extractors/principle-extractor.md",
        "case": CJ + "/extractors/case-extractor.md",
        "counter": CJ + "/extractors/counter-example-extractor.md",
        "glossary": CJ + "/extractors/glossary-extractor.md",
    }

    results = {}
    for name, path in extractors.items():
        if os.path.exists(path):
            extractor = open(path).read()
            prompt = f"""基于以下内容，提取 {name} 类型的方法论单元：

内容标题: {title}

---
{source_text[:6000]}
---

{extractor}"""
            results[name] = call_llm(source_text[:6000], extractor)
        else:
            results[name] = ""

    return results


def stage2_triple_verify(candidates):
    """阶段2: 三重验证筛选"""
    log("Stage 2: 三重验证...")
    verify_doc = open(CJ + "/methodology/03-stage1.5-triple-verify.md").read()

    verified = []
    for item in candidates:
        prompt = f"""验证以下候选方法论是否通过三重检验：

{item}

{verify_doc}

输出格式：
- 通过: [PASS] + 简短原因
- 不通过: [FAIL] + 原因"""
        result = call_llm(item, verify_doc)
        if "[PASS]" in result:
            verified.append(item)

    log(f"  三重验证通过率: {len(verified)}/{len(candidates)}")
    return verified


def stage3_ria_plus(verified_items, title):
    """阶段3: RIA++ 构造"""
    log("Stage 3: RIA++ 构造...")
    ria_doc = open(CJ + "/methodology/04-stage2-ria-plus.md").read()

    skills = []
    for item in verified_items:
        prompt = f"""将以下方法论构造为 RIA++ 结构：

原始内容: {title}

{item}

{ria_doc}

输出格式（按RIA++模板）：
## Skill名称

### R（原文引用）
### I（自己话重写）
### A1（书中案例）
### A2（未来触发场景）
### E（可执行步骤）
### B（边界与盲点）"""
        result = call_llm(item, ria_doc)
        skills.append(result)

    return skills


def call_llm(user_msg, system_msg="", model="minimaxai/minimax-m2.7"):
    """调用 NewAPI LLM"""
    import urllib.request

    payload = json.dumps({
        "model": model,
        "messages": [
            {"role": "system", "content": system_msg},
            {"role": "user", "content": user_msg},
        ],
        "max_tokens": 2000,
        "temperature": 0.7,
    }).encode()

    req = urllib.request.Request(
        "http://127.0.0.1:3000/v1/chat/completions",
        data=payload,
        headers={"Content-Type": "application/json", "Authorization": "Bearer 0ExNiLblJvIWBDpkS50fwOBw4MmqLyKdHJK5iQtlw9dOMWBP"},
    )

    try:
        with urllib.request.urlopen(req, timeout=60) as resp:
            data = json.load(resp)
            return data["choices"][0]["message"]["content"]
    except Exception as e:
        return f"Error: {e}"


def distill(source_text, title, output_dir=None):
    """完整蒸馏流程"""
    slug = re.sub(r'[^\w\u4e00-\u9fff]+', '_', title)[:40]
    out = output_dir or (OUTPUT + "/" + slug)
    os.makedirs(out, exist_ok=True)

    log(f"开始蒸馏: {title} -> {out}")

    # Stage 0: 整体理解
    overview = stage0_overview(source_text, title)
    with open(out + "/BOOK_OVERVIEW.md", "w") as f:
        f.write(f"# {title}\n\n{overview}\n")

    # Stage 1: 并行提取
    candidates = stage1_extract(source_text, title)

    # 展平候选
    all_candidates = []
    for name, content in candidates.items():
        if content:
            # 简单切分段落为候选
            for chunk in content.split("\n\n"):
                if len(chunk) > 50:
                    all_candidates.append(chunk.strip())

    # Stage 2: 三重验证
    verified = stage2_triple_verify(all_candidates[:20])  # 限制数量

    # Stage 3: RIA++ 构造
    skills = stage3_ria_plus(verified, title)

    # 保存 skills
    os.makedirs(out + "/skills", exist_ok=True)
    for i, skill in enumerate(skills, 1):
        with open(f"{out}/skills/skill_{i:03d}.md", "w") as f:
            f.write(skill)

    # 生成 INDEX
    index = f"# {title} - 技能地图\n\n"
    index += f"生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M')}\n\n"
    index += f"共 {len(skills)} 个验证通过的 skill\n\n"
    for i in range(len(skills)):
        index += f"- Skill {i+1}: 见 skills/skill_{i+1:03d}.md\n"

    with open(out + "/INDEX.md", "w") as f:
        f.write(index)

    log(f"蒸馏完成! 产出 {len(skills)} 个 skill -> {out}")
    return out


if __name__ == "__main__":
    cmd = sys.argv[1] if len(sys.argv) > 1 else "help"

    if cmd == "distill":
        if len(sys.argv) < 4:
            print("用法: cangjie_distill.py distill <source_text_file> <title>")
            sys.exit(1)
        text_file, title = sys.argv[2], sys.argv[3]
        with open(text_file) as f:
            source = f.read()
        distill(source, title)
    elif cmd == "help":
        print(__doc__)
    else:
        print(__doc__)