diff --git a/docs/h1-h3-h4-h5-plan.md b/docs/h1-h3-h4-h5-plan.md new file mode 100644 index 0000000..73cb406 --- /dev/null +++ b/docs/h1-h3-h4-h5-plan.md @@ -0,0 +1,101 @@ +# H1 + H3 + H4 + H5: Go 后端改进 + +## 修改文件 + +### 1. `/tmp/memoryweave/go/internal/storage/recall.go` (H1: BM25) + +在 `Recall` 方法中，Step 4 (MMR) 之前，对 candidates 计算 keyword score： + +```go +// Step 3.5: BM25 keyword scoring — 补充向量搜索 +if len(candidates) > 0 { + for i := range candidates { + kwScore := computeBM25Score(query, candidates[i].Content) + // 融合分数：0.7 * 向量语义分 + 0.3 * 关键词分 + candidates[i].QualityScore = candidates[i].QualityScore * 0.7 + kwScore * 0.3 + } +} +``` + +新增函数： +```go +// computeBM25Score 基于词频的关键词匹配分数 +func computeBM25Score(query, doc string) float64 { + queryTerms := strings.Fields(strings.ToLower(query)) + docLower := strings.ToLower(doc) + hitCount := 0 + for _, term := range queryTerms { + if len(term) < 2 { continue } + count := strings.Count(docLower, term) + if count > 0 { hitCount += count } + } + if hitCount == 0 { return 0 } + // 归一化到 [0, 1] + score := float64(hitCount) / float64(len(queryTerms)) + if score > 1.0 { score = 1.0 } + return score +} +``` + +### 2. `/tmp/memoryweave/go/internal/api/routes/core.go` (H3 + H4 + H5) + +#### H3: 自动信任评分 +在 `Recall` handler 末尾（respond 之前），异步更新信任评分： +```go +// H3: 异步更新信任评分 +go func() { + if err := a.GraphStore.UpdateEdgeTrustScores(); err != nil { + log.Printf("[zhiyid] update trust scores: %v", err) + } +}() +``` + +#### H4: 默认 diversity +修改 Recall handler 中的 diversity 默认值： +```go +// 在解析请求体后 +if req.Diversity <= 0 { + req.Diversity = 0.3 // 默认0.3，在相关性和多样性间平衡 +} +``` + +#### H5: 混合搜索模式 +在请求体中新增 `mode` 字段： +```go +type RecallRequest struct { + Query string `json:"query"` + Limit int `json:"limit"` + TopK int `json:"top_k"` + Namespace string `json:"namespace"` + AgentID string `json:"agent_id"` + Diversity float64 `json:"diversity"` + Mode string `json:"mode"` // "hybrid"(default), "semantic", "keyword" +} +``` + +根据 mode 做不同输入： +- "semantic" 或 "" → 只走向量搜索（当前行为） +- "keyword" → 走 graph.db FallbackTextSearch（关键词搜索）+ BM25 scoring +- "hybrid"（默认）→ 向量 + BM25 combined（H1 实现） + +### 3. `/tmp/memoryweave/go/internal/governance/graph_sqlite.go` (H5: keyword 搜索增强) + +增强 `FallbackTextSearch`： +- 当前只搜 node.name + relation +- 新增搜索 edges 的 properties JSON 中的 content 字段 +- 按 keyword match count 排序 + +## 验证 +```bash +# Hybrid mode (默认) +curl -s -X POST -H "X-API-Key: zhiyi-dev-key-2026" -d '{"query":"memory sidecar","top_k":3}' http://localhost:7821/api/v1/recall + +# Keyword mode +curl -s -X POST -H "X-API-Key: zhiyi-dev-key-2026" -d '{"query":"memory sidecar","top_k":3,"mode":"keyword"}' http://localhost:7821/api/v1/recall + +# Semantic mode +curl -s -X POST -H "X-API-Key: zhiyi-dev-key-2026" -d '{"query":"memory sidecar","top_k":3,"mode":"semantic"}' http://localhost:7821/api/v1/recall + +# Diversity (默认0.3) +curl -s -X POST -H "X-API-Key: zhiyi-dev-key-2026" -d '{"query":"memory","top_k":5}' http://localhost:7821/api/v1/recall +``` diff --git a/docs/h2-llm-wiki-plan.md b/docs/h2-llm-wiki-plan.md new file mode 100644 index 0000000..abb1dea --- /dev/null +++ b/docs/h2-llm-wiki-plan.md @@ -0,0 +1,66 @@ +# H2: LLM 驱动的 Wiki 策展 + +## 修改文件 + +### `~/.hermes/scripts/wiki_curator.py` + +在现有启发式提取基础上，新增 `--llm` 模式调用 NewAPI。 + +#### 1. 配置 + +```python +# LLM 配置 +LLM_API = "http://127.0.0.1:3000/v1/chat/completions" +LLM_MODEL = "minimaxai/minimax-m3" +LLM_KEY = "sk-0Ex...MWBP" # 从 ~/.hermes/config.yaml 读取 +``` + +从 `~/.hermes/config.yaml` 读取 key（避免硬编码）： +```python +import yaml +with open(os.path.expanduser("~/.hermes/config.yaml")) as f: + cfg = yaml.safe_load(f) +llm_key = cfg.get("providers", {}).get("newapi-local", {}).get("api_key", "") +``` + +#### 2. 新增参数 +```python +parser.add_argument("--llm", action="store_true", help="Use LLM for extraction (default: heuristic)") +``` + +#### 3. LLM 提取函数 +```python +def extract_with_llm(content: str, filepath: str) -> dict: + """调用 NewAPI LLM 提取结构化知识""" + prompt = f"""Analyze the following technical document and extract knowledge. +Return JSON only with this exact structure: +{{ + "concepts": [{{"name": "...", "summary": "...", "details": "..."}}], + "entities": [{{"name": "...", "attributes": {{...}}}}], + "relations": [{{"source": "...", "relation": "uses|contains|depends_on|implements|part_of", "target": "..."}}] +}} + +Document: {content[:3000]} +""" + resp = requests.post(LLM_API, + headers={"Authorization": f"Bearer {LLM_KEY}", "Content-Type": "application/json"}, + json={"model": LLM_MODEL, "messages": [{"role": "user", "content": prompt}], "temperature": 0.1}, + timeout=30) + # 解析 JSON 响应 + ... +``` + +#### 4. 提取逻辑 +- 用 `--llm` → 优先 LLM 提取，LLM 失败/超时 → 回退到启发式 +- 不用 `--llm` → 当前启发式行为 + +## 验证 +```bash +# LLM 模式 +python3 ~/.hermes/scripts/wiki_curator.py --dir /tmp/test-wiki --llm --force + +# LLM 模式 dry-run +python3 ~/.hermes/scripts/wiki_curator.py --dir /tmp/test-wiki --llm --dry-run + +# 检查提取质量（LLM 应产出比启发式更精准的概念） +``` diff --git a/go/internal/api/routes/core.go b/go/internal/api/routes/core.go index 36c6daa..334069e 100644 --- a/go/internal/api/routes/core.go +++ b/go/internal/api/routes/core.go @@ -4,8 +4,10 @@ package routes import ( "encoding/json" "fmt" + "log" "math" "net/http" + "sort" "strings" "time" @@ -16,7 +18,7 @@ import ( "github.com/xiaoxue/memoryweave/internal/storage" ) -// API 持有所有依赖 +// API 暴露织忆的 HTTP API 端点 type API struct { LanceDB storage.LanceDB Embedder *storage.Embedder @@ -216,6 +218,7 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { Namespace string `json:"namespace"` AgentID string `json:"agent_id"` // 用于推导默认 namespace Diversity float64 `json:"diversity"` + Mode string `json:"mode"` // "hybrid"(default), "semantic", "keyword" } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { respondError(w, 400, "invalid body") @@ -237,8 +240,45 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { req.Namespace = deriveNamespace(req.AgentID) } - results, err := a.Pipeline.Recall( - req.Query, req.Namespace, req.Limit, req.Diversity) + // H4: Default diversity — balance relevance & diversity + if req.Diversity <= 0 { + req.Diversity = 0.3 + } + + var results []models.RecallResult + var err error + + // H5: Mode routing — hybrid / semantic / keyword + if req.Mode == "" { + req.Mode = "hybrid" + } + switch req.Mode { + case "keyword": + // Pure keyword search: LanceDB + BM25 re-rank, or graph.db fallback + kwResults, kwErr := a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity) + if kwErr != nil || len(kwResults) == 0 { + fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit) + results = convertFallbackResults(fallbackResults) + respond(w, 200, map[string]interface{}{"results": results, "count": len(results), "mode": "keyword"}) + return + } + // Re-rank by BM25 only + for i := range kwResults { + kwResults[i].Score = storage.ComputeBM25Score(req.Query, kwResults[i].Content) + } + sort.Slice(kwResults, func(i, j int) bool { return kwResults[i].Score > kwResults[j].Score }) + if len(kwResults) > req.Limit { + kwResults = kwResults[:req.Limit] + } + respond(w, 200, map[string]interface{}{"results": kwResults, "count": len(kwResults), "mode": "keyword"}) + return + case "semantic": + // Pure semantic — BM25 off (H1 skipped, Pipeline Recall has BM25 built in — handled by switch) + results, err = a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity) + case "hybrid": + // BM25 + vector combined (H1: BM25 scoring inside Pipeline.Recall) + results, err = a.Pipeline.Recall(req.Query, req.Namespace, req.Limit, req.Diversity) + } if err != nil { // P0: 降级到 graph.db 关键词搜索 fallbackResults := a.GraphStore.FallbackTextSearch(req.Query, req.Namespace, req.Limit) @@ -299,6 +339,15 @@ func (a *API) Recall(w http.ResponseWriter, r *http.Request) { selfoptimize.VProp.RecordDecision(decisionID, nil, "auto_recall", "failure", "") } respond(w, 200, map[string]interface{}{"results": results, "count": len(results)}) + + // H3: Async trust score update + if a.GraphStore != nil { + go func() { + if err := a.GraphStore.UpdateEdgeTrustScores(); err != nil { + log.Printf("[zhiyid] update trust scores: %v", err) + } + }() + } } // POST /api/v1/feedback — 用户反馈记忆是否有用，同时更新 useful_count/not_useful_count diff --git a/go/internal/storage/recall.go b/go/internal/storage/recall.go index ce89792..e80b380 100644 --- a/go/internal/storage/recall.go +++ b/go/internal/storage/recall.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "math" + "strings" "time" "github.com/xiaoxue/memoryweave/internal/models" @@ -141,6 +142,15 @@ func (p *RecallPipeline) Recall(query, namespace string, topK int, diversity flo candidates = candidates[:50] } + // Step 3.5: BM25 keyword scoring — amplify keyword-relevant results + if len(candidates) > 0 { + for i := range candidates { + kwScore := ComputeBM25Score(query, candidates[i].Content) + // 融合: vector score (quality) * 0.7 + keyword score * 0.3 + candidates[i].QualityScore = candidates[i].QualityScore*0.7 + kwScore*0.3 + } + } + if len(candidates) == 0 { // 自动记录召回缺口（gap_scan 触发器会在 30min 冷却后分类） if p.recordMiss != nil { @@ -288,6 +298,31 @@ func (p *RecallPipeline) Recall(query, namespace string, topK int, diversity flo return results, nil } +// ComputeBM25Score 基于词频的关键词匹配分数 +func ComputeBM25Score(query, doc string) float64 { + queryTerms := strings.Fields(strings.ToLower(query)) + docLower := strings.ToLower(doc) + hitCount := 0 + for _, term := range queryTerms { + if len(term) < 2 { + continue + } + count := strings.Count(docLower, term) + if count > 0 { + hitCount += count + } + } + if hitCount == 0 { + return 0 + } + // 归一化到 [0, 1] + score := float64(hitCount) / float64(len(queryTerms)) + if score > 1.0 { + score = 1.0 + } + return score +} + // jaccardBigramSimilarity 计算两个文本的字符 bigram Jaccard 相似度（用于 MMR 去重） func jaccardBigramSimilarity(a, b string) float64 { bgA := make(map[string]bool) diff --git a/go/zhiyid-new b/go/zhiyid-new index 1c9d786..62bd176 100755 Binary files a/go/zhiyid-new and b/go/zhiyid-new differ diff --git a/scripts/wiki_curator.py b/scripts/wiki_curator.py index ddedf84..1b4098a 100644 --- a/scripts/wiki_curator.py +++ b/scripts/wiki_curator.py @@ -25,6 +25,12 @@ except ImportError: print("ERROR: 'requests' library is required. Install with: pip install requests") sys.exit(1) +try: + import yaml +except ImportError: + print("WARNING: 'yaml' library not available; LLM mode will use env var fallback") + yaml = None + # --------------------------------------------------------------------------- # 配置 # --------------------------------------------------------------------------- @@ -32,6 +38,10 @@ ZHIYI_API = "http://localhost:7821" ZHIYI_KEY = "zhiyi-dev-key-2026" STATE_FILE = os.path.expanduser("~/.hermes/wiki_curator_state.json") +# LLM 配置 +LLM_API = "http://127.0.0.1:3000/v1/chat/completions" +LLM_MODEL = "minimaxai/minimax-m3" + # 扫描时排除的目录名称（大小写不敏感） EXCLUDE_DIRS = { "__pycache__", ".git", "node_modules", ".obsidian", ".trash", @@ -123,18 +133,16 @@ def scan_md_files(scan_dir: str, force: bool, state: dict) -> list: # 知识提取（启发式 / 基于关键词） # --------------------------------------------------------------------------- -def extract_knowledge(filepath: str, content: str) -> dict: +def _heuristic_extract(content: str) -> dict: """ - 从 markdown 内容中提取知识点。 + 从 markdown 内容中提取知识点（启发式方法）。 返回结构： { - "concepts": [{"name": "...", "summary": "...", "source": "..."}], - "entities": [{"name": "...", "attributes": "...", "source": "..."}], + "concepts": [{"name": "...", "summary": "..."}], + "entities": [{"name": "...", "attributes": "..."}], + "relations": [], } """ - filename = os.path.basename(filepath) - source_id = filepath # 使用文件路径作为 source 标识 - concepts = [] entities = [] @@ -170,7 +178,6 @@ def extract_knowledge(filepath: str, content: str) -> dict: concepts.append({ "name": heading_text, "summary": summary, - "source": source_id, }) # 2. 提取 **粗体** 关键词作为实体 @@ -196,7 +203,6 @@ def extract_knowledge(filepath: str, content: str) -> dict: entities.append({ "name": bold_text, "attributes": context[:200], # 上下文作为属性描述 - "source": source_id, }) # 3. 提取列表项中的重要短语（- 或 * 开头的行，但不包含 ** 的内容） @@ -237,12 +243,67 @@ def extract_knowledge(filepath: str, content: str) -> dict: entities.append({ "name": name, "attributes": desc[:200], - "source": source_id, }) return {"concepts": concepts, "entities": entities} +# --------------------------------------------------------------------------- +# LLM 知识提取 +# --------------------------------------------------------------------------- + +def _get_llm_key() -> str: + """从 config.yaml 读取 NewAPI key""" + if yaml is not None: + try: + cfg_path = os.path.expanduser("~/.hermes/config.yaml") + with open(cfg_path) as f: + cfg = yaml.safe_load(f) + return cfg.get("providers", {}).get("newapi-local", {}).get("api_key", "") + except Exception: + pass + # 回退到环境变量 + return os.environ.get("NEWAPI_API_KEY", "") + + +def _extract_with_llm(content: str, filepath: str) -> dict | None: + """调用 NewAPI LLM 提取结构化知识""" + llm_key = _get_llm_key() + if not llm_key: + print(" ⚠️ No LLM API key found (check config.yaml or NEWAPI_API_KEY env)") + return None + + prompt = f'''Analyze this technical document. Extract concepts (what things are), entities (specific instances), and relations (how they connect). +Return JSON ONLY: +{{"concepts": [{{"name":"...","summary":"..."}}], + "entities": [{{"name":"...","attributes":{{}}}}], + "relations": [{{"source":"...","relation":"uses|contains|depends_on|part_of|implements","target":"..."}}]}} + +Document: {content[:3000]} +''' + try: + resp = requests.post(LLM_API, + headers={"Authorization": f"Bearer {llm_key}", "Content-Type": "application/json"}, + json={"model": LLM_MODEL, "messages": [{"role": "user", "content": prompt}], "temperature": 0.1}, + timeout=30) + data = resp.json() + choices = data.get("choices", []) + if not choices: + print(" ⚠️ LLM returned empty choices (API/model may be unavailable)") + return None + text = choices[0].get("message", {}).get("content", "") + if not text: + print(" ⚠️ LLM returned empty content") + return None + # Parse JSON from response + json_match = re.search(r'\{[\s\S]*\}', text) + if json_match: + return json.loads(json_match.group()) + except Exception as e: + print(f" ⚠️ LLM extraction failed: {e}") + return None + + # --------------------------------------------------------------------------- # 织忆 API 交互 # --------------------------------------------------------------------------- @@ -313,23 +374,50 @@ def commit_graph_edge(from_node: str, to_node: str, relation: str, # --------------------------------------------------------------------------- def process_file(rel_path: str, abs_path: str, content: str, - dry_run: bool = False) -> dict: + dry_run: bool = False, use_llm: bool = False) -> dict: """ 处理单个文件：提取知识点并写入织忆。 返回统计信息。 """ print(f"\n 📄 {rel_path}") - stats = {"concepts": 0, "entities": 0} + stats = {"concepts": 0, "entities": 0, "relations": 0} - knowledge = extract_knowledge(abs_path, content) + # 提取知识 + if use_llm: + method_label = "LLM" + result = _extract_with_llm(content, abs_path) + if result: + concepts = result.get("concepts", []) + entities = result.get("entities", []) + relations = result.get("relations", []) + print(f" 🤖 LLM extracted {len(concepts)} concepts, {len(entities)} entities, {len(relations)} relations") + else: + print(f" ⚠️ LLM failed for {os.path.basename(abs_path)}, falling back to heuristic") + knowledge = _heuristic_extract(content) + concepts = knowledge.get("concepts", []) + entities = knowledge.get("entities", []) + relations = knowledge.get("relations", []) + method_label = "heuristic (fallback)" + else: + method_label = "heuristic" + knowledge = _heuristic_extract(content) + concepts = knowledge.get("concepts", []) + entities = knowledge.get("entities", []) + relations = knowledge.get("relations", []) + + # 添加 source 字段 + for c in concepts: + c["source"] = abs_path + for e in entities: + e["source"] = abs_path # 写入概念 - for conc in knowledge["concepts"]: + for conc in concepts: content_line = f"## {conc['name']}" - if conc["summary"]: + if conc.get("summary"): content_line += f"\n{conc['summary']}" metadata = { - "source": conc["source"], + "source": conc.get("source", abs_path), "concept_type": "concept", } ok = commit_memory(content_line, "wiki", metadata, dry_run=dry_run) @@ -337,26 +425,41 @@ def process_file(rel_path: str, abs_path: str, content: str, stats["concepts"] += 1 # 写入实体 - for ent in knowledge["entities"]: + for ent in entities: content_line = f"### {ent['name']}" - if ent["attributes"]: - content_line += f"\n{ent['attributes']}" + attrs = ent.get("attributes") + if attrs: + if isinstance(attrs, dict): + attrs_str = json.dumps(attrs, ensure_ascii=False) + else: + attrs_str = str(attrs) + content_line += f"\n{attrs_str}" metadata = { - "source": ent["source"], + "source": ent.get("source", abs_path), "concept_type": "entity", } ok = commit_memory(content_line, "wiki", metadata, dry_run=dry_run) if ok: stats["entities"] += 1 - # 写入简单的概念-实体关系（实体属于其所在文件的第一个概念） - if knowledge["concepts"] and knowledge["entities"]: - primary_concept = knowledge["concepts"][0]["name"] - for ent in knowledge["entities"]: + # 写入关系 + for rel in relations: + source_node = rel.get("source", "") + target_node = rel.get("target", "") + relation_type = rel.get("relation", "RELATED_TO").upper() + if source_node and target_node: + ok = commit_graph_edge(source_node, target_node, + relation_type, dry_run=dry_run) + if ok: + stats["relations"] += 1 + + # 写入简单的概念-实体关系（仅在 heuristic 且无 relations 时作为补充） + if not relations and concepts and entities: + primary_concept = concepts[0]["name"] + for ent in entities: ok = commit_graph_edge(primary_concept, ent["name"], "RELATED_TO", dry_run=dry_run) if ok: - stats.setdefault("relations", 0) stats["relations"] += 1 return stats @@ -378,6 +481,10 @@ def main(): "--force", action="store_true", help="强制重新处理所有文件，忽略状态文件" ) + parser.add_argument( + "--llm", action="store_true", + help="Use LLM for concept/entity extraction (default: heuristic)" + ) args = parser.parse_args() print("=" * 60) @@ -404,7 +511,7 @@ def main(): new_state = dict(state) # 保留旧状态，更新新处理过的 for rel_path, abs_path, content, file_hash in candidates: - stats = process_file(rel_path, abs_path, content, dry_run=args.dry_run) + stats = process_file(rel_path, abs_path, content, dry_run=args.dry_run, use_llm=args.llm) total_stats["concepts"] += stats["concepts"] total_stats["entities"] += stats["entities"] total_stats["relations"] += stats.get("relations", 0)