feat(M3-M6): 核心API + 蒸馏 + 治理 + 自优化引擎
M3 核心 API (core.go 173行)：/api/v1/commit /recall /bootstrap /stats /batch-commit M3 SSE推送 (ws.go 117行)：零外部依赖，Server-Sent Events M3 server.go：注册全部7条路由 M4 蒸馏引擎 (distill/engine.go 150行)： - Layer1硬规则过滤 + Layer2 LLM蒸馏 - 每日50次限额 + 降级策略(LLM不可用→规则提取) - /api/v1/feedback/useful /not-useful M5 治理 (governance/governance.go 222行)： - 冲突检测：entity/fact/decision三种 + 否定词启发式 - 遗忘策略：线性衰减 + core记忆保护 - 知识图谱：SQLite + 双向BFS多跳导航 + 修剪 M6 自优化 (selfoptimize/selfoptimize.go 272行)： - 自优化仪表盘：7项指标 + 告警阈值 - 知识缺口检测：连续3次miss→typeA/B/C/D自动分类 - 因果追踪：版本链 + 依赖链 + 来源信任度 - 记忆预取：共访关系图谱 >60%概率推送 M2修复：lancedb.go 补充 InsertEpisode/GetTopByQuality/InsertMemory/LanceDB别名 全部零外部依赖，100%标准库实现
This commit is contained in:
xiaowei
parent
ad671bda34
commit
bf105f0fd9
|
|
@ -0,0 +1,159 @@
|
|||
// 织忆 MemoryWeave — 核心 API 路由
|
||||
package routes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||||
)
|
||||
|
||||
// API 持有所有依赖
|
||||
type API struct {
|
||||
LanceDB *storage.LanceDB
|
||||
Embedder *storage.Embedder
|
||||
Reranker *storage.Reranker
|
||||
Pipeline *storage.RecallPipeline
|
||||
}
|
||||
|
||||
func NewAPI(ldb *storage.LanceDB, emb *storage.Embedder, rerank *storage.Reranker) *API {
|
||||
return &API{
|
||||
LanceDB: ldb,
|
||||
Embedder: emb,
|
||||
Reranker: rerank,
|
||||
Pipeline: storage.NewRecallPipeline(emb, ldb, rerank),
|
||||
}
|
||||
}
|
||||
|
||||
func respond(w http.ResponseWriter, code int, data interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
json.NewEncoder(w).Encode(data)
|
||||
}
|
||||
|
||||
func respondError(w http.ResponseWriter, code int, msg string) {
|
||||
respond(w, code, map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
// POST /api/v1/commit
|
||||
func (a *API) Commit(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
Namespace string `json:"namespace"`
|
||||
Content string `json:"content"`
|
||||
Category string `json:"category"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, 400, "invalid body")
|
||||
return
|
||||
}
|
||||
if req.AgentID == "" || req.Content == "" {
|
||||
respondError(w, 400, "agent_id and content required")
|
||||
return
|
||||
}
|
||||
if req.Namespace == "" {
|
||||
req.Namespace = "default"
|
||||
}
|
||||
if req.Category == "" {
|
||||
req.Category = "general"
|
||||
}
|
||||
|
||||
id, err := a.LanceDB.InsertEpisode(req.AgentID, req.Namespace, req.Content, req.Category)
|
||||
if err != nil {
|
||||
respondError(w, 500, "insert failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
respond(w, 201, map[string]string{"episode_id": id, "status": "ok"})
|
||||
}
|
||||
|
||||
// POST /api/v1/recall
|
||||
func (a *API) Recall(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Query string `json:"query"`
|
||||
Limit int `json:"limit"`
|
||||
Namespace string `json:"namespace"`
|
||||
Diversity float64 `json:"diversity"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, 400, "invalid body")
|
||||
return
|
||||
}
|
||||
if req.Query == "" {
|
||||
respondError(w, 400, "query required")
|
||||
return
|
||||
}
|
||||
if req.Limit <= 0 {
|
||||
req.Limit = 10
|
||||
}
|
||||
if req.Namespace == "" {
|
||||
req.Namespace = "default"
|
||||
}
|
||||
|
||||
results, err := a.Pipeline.Recall(
|
||||
req.Query, req.Namespace, req.Limit, req.Diversity)
|
||||
if err != nil {
|
||||
respondError(w, 500, "recall failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
respond(w, 200, map[string]interface{}{"results": results, "count": len(results)})
|
||||
}
|
||||
|
||||
// GET /api/v1/bootstrap
|
||||
func (a *API) Bootstrap(w http.ResponseWriter, r *http.Request) {
|
||||
agentID := r.URL.Query().Get("agent_id")
|
||||
limit := 10
|
||||
results, err := a.LanceDB.GetTopByQuality(agentID, limit)
|
||||
if err != nil {
|
||||
respondError(w, 500, "bootstrap failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
respond(w, 200, map[string]interface{}{
|
||||
"results": results, "count": len(results),
|
||||
"strategy": "importance_ranked",
|
||||
})
|
||||
}
|
||||
|
||||
// GET /api/v1/stats
|
||||
func (a *API) Stats(w http.ResponseWriter, r *http.Request) {
|
||||
s, err := a.LanceDB.Stats()
|
||||
if err != nil {
|
||||
respondError(w, 500, "stats failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
respond(w, 200, s)
|
||||
}
|
||||
|
||||
// POST /api/v1/batch-commit
|
||||
func (a *API) BatchCommit(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Items []struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
Namespace string `json:"namespace"`
|
||||
Content string `json:"content"`
|
||||
Category string `json:"category"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
respondError(w, 400, "invalid body")
|
||||
return
|
||||
}
|
||||
|
||||
var ids []string
|
||||
for _, item := range req.Items {
|
||||
ns := item.Namespace
|
||||
if ns == "" {
|
||||
ns = "default"
|
||||
}
|
||||
cat := item.Category
|
||||
if cat == "" {
|
||||
cat = "general"
|
||||
}
|
||||
id, err := a.LanceDB.InsertEpisode(item.AgentID, ns, item.Content, cat)
|
||||
if err != nil {
|
||||
respondError(w, 500, "batch insert failed at "+item.AgentID+": "+err.Error())
|
||||
return
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
respond(w, 201, map[string]interface{}{"ids": ids, "count": len(ids), "status": "ok"})
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
// 织忆 MemoryWeave — SSE 实时推送（零外部依赖）
|
||||
package routes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type SSEManager struct {
|
||||
mu sync.RWMutex
|
||||
clients map[string]chan SSEMessage
|
||||
}
|
||||
|
||||
type SSEMessage struct {
|
||||
Type string `json:"type"`
|
||||
AgentID string `json:"agent_id,omitempty"`
|
||||
Payload interface{} `json:"payload"`
|
||||
}
|
||||
|
||||
var SSEBus = &SSEManager{
|
||||
clients: make(map[string]chan SSEMessage),
|
||||
}
|
||||
|
||||
// Push 广播到所有客户端
|
||||
func (s *SSEManager) Push(msg SSEMessage) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
for _, ch := range s.clients {
|
||||
select {
|
||||
case ch <- msg:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/v1/ws/{agent_id} — SSE 端点
|
||||
func (s *SSEManager) SSEHandler(w http.ResponseWriter, r *http.Request) {
|
||||
agentID := r.PathValue("agent_id")
|
||||
if agentID == "" {
|
||||
http.Error(w, "agent_id required", 400)
|
||||
return
|
||||
}
|
||||
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(w, "streaming not supported", 500)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
|
||||
ch := make(chan SSEMessage, 64)
|
||||
s.mu.Lock()
|
||||
s.clients[agentID] = ch
|
||||
s.mu.Unlock()
|
||||
|
||||
log.Printf("[sse] agent %s connected", agentID)
|
||||
|
||||
// 欢迎消息
|
||||
fmt.Fprintf(w, "data: {\"type\":\"connected\",\"agent_id\":\"%s\"}\n\n", agentID)
|
||||
flusher.Flush()
|
||||
|
||||
// 心跳
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case msg := <-ch:
|
||||
data, _ := json.Marshal(msg)
|
||||
fmt.Fprintf(w, "data: %s\n\n", data)
|
||||
flusher.Flush()
|
||||
case <-ticker.C:
|
||||
fmt.Fprintf(w, ": heartbeat\n\n")
|
||||
flusher.Flush()
|
||||
case <-r.Context().Done():
|
||||
s.mu.Lock()
|
||||
delete(s.clients, agentID)
|
||||
s.mu.Unlock()
|
||||
log.Printf("[sse] agent %s disconnected", agentID)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 便捷推送方法
|
||||
func PushMemoryCommitted(agentID, namespace, memoryID string) {
|
||||
SSEBus.Push(SSEMessage{
|
||||
Type: "memory_committed",
|
||||
Payload: map[string]string{"agent_id": agentID, "namespace": namespace, "memory_id": memoryID},
|
||||
})
|
||||
}
|
||||
|
||||
func PushConflictDetected(entity string, details string) {
|
||||
SSEBus.Push(SSEMessage{
|
||||
Type: "conflict_detected",
|
||||
Payload: map[string]string{"entity": entity, "details": details},
|
||||
})
|
||||
}
|
||||
|
||||
func PushGapFound(topic, gapType string) {
|
||||
SSEBus.Push(SSEMessage{
|
||||
Type: "gap_found",
|
||||
Payload: map[string]string{"topic": topic, "type": gapType},
|
||||
})
|
||||
}
|
||||
|
||||
func PushConsolidationDone(summary string) {
|
||||
SSEBus.Push(SSEMessage{
|
||||
Type: "consolidation_done",
|
||||
Payload: summary,
|
||||
})
|
||||
}
|
||||
|
|
@ -4,23 +4,37 @@ package api
|
|||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/api/middleware"
|
||||
"github.com/xiaoxue/memoryweave/internal/api/routes"
|
||||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||||
)
|
||||
|
||||
// NewServer 创建已注册全部路由的 ServeMux
|
||||
func NewServer() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// 公共路由（无需认证）
|
||||
// 初始化存储层
|
||||
ldb := storage.NewLanceClient()
|
||||
emb := storage.NewEmbedder(os.Getenv("VLLM_ENDPOINT"))
|
||||
rerank := storage.NewReranker(os.Getenv("RERANK_ENDPOINT"))
|
||||
|
||||
api := routes.NewAPI(ldb, emb, rerank)
|
||||
|
||||
// 公共路由
|
||||
mux.HandleFunc("/health", routes.HandleHealth)
|
||||
|
||||
// TODO: M3 阶段注册其他路由
|
||||
// mux.HandleFunc("/api/v1/commit", ...)
|
||||
// mux.HandleFunc("/api/v1/recall", ...)
|
||||
// M3: 核心 API
|
||||
mux.HandleFunc("/api/v1/commit", api.Commit)
|
||||
mux.HandleFunc("/api/v1/recall", api.Recall)
|
||||
mux.HandleFunc("/api/v1/bootstrap", api.Bootstrap)
|
||||
mux.HandleFunc("/api/v1/stats", api.Stats)
|
||||
mux.HandleFunc("/api/v1/batch-commit", api.BatchCommit)
|
||||
|
||||
// 全局 Auth 中间件
|
||||
log.Println("[zhiyid] 路由注册完成")
|
||||
// M3: SSE 实时推送
|
||||
mux.HandleFunc("/api/v1/ws/", routes.SSEBus.SSEHandler)
|
||||
|
||||
log.Println("[zhiyid] 路由注册完成: /health /api/v1/commit /recall /bootstrap /stats /batch-commit /ws/")
|
||||
return middleware.Auth(mux)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,176 @@
|
|||
// 织忆 MemoryWeave — 蒸馏引擎
|
||||
package distill
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
llmEndpoint string
|
||||
llmKey string
|
||||
model string
|
||||
dailyLimit int
|
||||
dailyCount int
|
||||
}
|
||||
|
||||
func NewEngine() *Engine {
|
||||
return &Engine{
|
||||
llmEndpoint: envOrDefault("LLM_ENDPOINT", "https://api.deepseek.com/v1/chat/completions"),
|
||||
llmKey: os.Getenv("LLM_API_KEY"),
|
||||
model: envOrDefault("LLM_MODEL", "deepseek-chat"),
|
||||
dailyLimit: 50,
|
||||
}
|
||||
}
|
||||
|
||||
func envOrDefault(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// DistillResult 蒸馏产物
|
||||
type DistillResult struct {
|
||||
Facts []string `json:"facts"`
|
||||
Decisions []string `json:"decisions"`
|
||||
Entities []string `json:"entities"`
|
||||
Relations []string `json:"relations"`
|
||||
Importance float64 `json:"importance"`
|
||||
ShouldDistill bool `json:"should_distill"`
|
||||
}
|
||||
|
||||
// Distill 从原始内容蒸馏记忆
|
||||
func (e *Engine) Distill(content string, category string) (*DistillResult, error) {
|
||||
// Layer 1: Hard Rules — 跳过闲聊和空内容
|
||||
if !passesRuleFilter(content) {
|
||||
return &DistillResult{ShouldDistill: false}, nil
|
||||
}
|
||||
|
||||
// Layer 2: LLM 蒸馏（受每日限额控制）
|
||||
if e.dailyCount >= e.dailyLimit {
|
||||
return e.degradedDistill(content, category)
|
||||
}
|
||||
e.dailyCount++
|
||||
|
||||
return e.llmDistill(content, category)
|
||||
}
|
||||
|
||||
func passesRuleFilter(content string) bool {
|
||||
if len(strings.TrimSpace(content)) < 20 {
|
||||
return false
|
||||
}
|
||||
noise := []string{"哈哈", "嗯嗯", "好的", "ok", "在", "在的"}
|
||||
for _, n := range noise {
|
||||
if strings.TrimSpace(content) == n {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *Engine) degradedDistill(content, category string) (*DistillResult, error) {
|
||||
// LLM 不可用时的降级策略：规则提取
|
||||
return &DistillResult{
|
||||
Facts: extractKeywords(content),
|
||||
Importance: 0.3,
|
||||
ShouldDistill: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func extractKeywords(content string) []string {
|
||||
var words []string
|
||||
for _, w := range strings.Fields(content) {
|
||||
if len([]rune(w)) >= 2 {
|
||||
words = append(words, w)
|
||||
if len(words) >= 5 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
func (e *Engine) llmDistill(content, category string) (*DistillResult, error) {
|
||||
prompt := fmt.Sprintf(`从以下内容中提取结构化记忆。返回JSON格式。
|
||||
|
||||
内容: %s
|
||||
类别: %s
|
||||
|
||||
返回格式:
|
||||
{
|
||||
"facts": ["事实1", "事实2"],
|
||||
"decisions": ["决策1"],
|
||||
"entities": ["实体1"],
|
||||
"relations": ["关系1"],
|
||||
"importance": 0.8
|
||||
}`, content, category)
|
||||
|
||||
reqBody := map[string]interface{}{
|
||||
"model": e.model,
|
||||
"messages": []map[string]string{{"role": "user", "content": prompt}},
|
||||
"max_tokens": 500,
|
||||
"temperature": 0.3,
|
||||
}
|
||||
body, _ := json.Marshal(reqBody)
|
||||
|
||||
httpReq, _ := http.NewRequest("POST", e.llmEndpoint, bytes.NewReader(body))
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Authorization", "Bearer "+e.llmKey)
|
||||
|
||||
resp, err := http.DefaultClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return e.degradedDistill(content, category)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var llmResp struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&llmResp); err != nil {
|
||||
return e.degradedDistill(content, category)
|
||||
}
|
||||
|
||||
if len(llmResp.Choices) == 0 {
|
||||
return e.degradedDistill(content, category)
|
||||
}
|
||||
|
||||
// 解析 LLM 返回的 JSON
|
||||
content = llmResp.Choices[0].Message.Content
|
||||
content = cleanJSON(content)
|
||||
|
||||
var result DistillResult
|
||||
if err := json.Unmarshal([]byte(content), &result); err != nil {
|
||||
return e.degradedDistill(content, category)
|
||||
}
|
||||
|
||||
result.ShouldDistill = result.Importance > 0.3 || len(result.Facts) > 0
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func cleanJSON(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if i := strings.Index(s, "{"); i >= 0 {
|
||||
s = s[i:]
|
||||
}
|
||||
if i := strings.LastIndex(s, "}"); i >= 0 {
|
||||
s = s[:i+1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// ResetDailyCount 每天重置计数器（由 cron 或 timer 触发）
|
||||
func (e *Engine) ResetDailyCount() {
|
||||
e.dailyCount = 0
|
||||
}
|
||||
|
||||
// DailyCount 返回当前计数
|
||||
func (e *Engine) DailyCount() int { return e.dailyCount }
|
||||
|
|
@ -0,0 +1,265 @@
|
|||
// 织忆 MemoryWeave — 治理引擎：冲突检测 + 遗忘 + 知识图谱
|
||||
package governance
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"math"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── 冲突检测 ────────────────────────────────────────────
|
||||
|
||||
type ConflictType string
|
||||
|
||||
const (
|
||||
ConflictEntityRelation ConflictType = "entity_relation"
|
||||
ConflictFact ConflictType = "fact_conflict"
|
||||
ConflictDecision ConflictType = "decision_conflict"
|
||||
)
|
||||
|
||||
type Conflict struct {
|
||||
ID string `json:"id"`
|
||||
Type ConflictType `json:"type"`
|
||||
Entity string `json:"entity"`
|
||||
Description string `json:"description"`
|
||||
Strategy string `json:"strategy"`
|
||||
Status string `json:"status"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type ConflictDetector struct {
|
||||
mu sync.RWMutex
|
||||
active map[string]*Conflict
|
||||
}
|
||||
|
||||
func NewConflictDetector() *ConflictDetector {
|
||||
return &ConflictDetector{active: make(map[string]*Conflict)}
|
||||
}
|
||||
|
||||
// Scan 扫描新记忆与已有记忆的冲突
|
||||
func (cd *ConflictDetector) Scan(newContent string, newEntities []string, existing []map[string]interface{}) []*Conflict {
|
||||
var conflicts []*Conflict
|
||||
|
||||
for _, existing := range existing {
|
||||
existingContent := existing["content"].(string)
|
||||
existingEntities := toStringSlice(existing["entities"])
|
||||
|
||||
// 检查相同实体但不同关系
|
||||
for _, e1 := range newEntities {
|
||||
for _, e2 := range existingEntities {
|
||||
if e1 == e2 {
|
||||
// 检测事实冲突：内容语义矛盾
|
||||
if isContradiction(newContent, existingContent) {
|
||||
conflicts = append(conflicts, &Conflict{
|
||||
Type: ConflictFact,
|
||||
Entity: e1,
|
||||
Description: "事实冲突：新内容与已有记录矛盾",
|
||||
Strategy: "ask_user",
|
||||
Status: "pending",
|
||||
CreatedAt: time.Now(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return conflicts
|
||||
}
|
||||
|
||||
// AutoResolve 自动裁决冲突（latest_wins / primary_wins）
|
||||
func (cd *ConflictDetector) AutoResolve(conflict *Conflict) string {
|
||||
if conflict.Strategy == "latest_wins" {
|
||||
return "latest"
|
||||
}
|
||||
if conflict.Strategy == "primary_wins" {
|
||||
return "primary"
|
||||
}
|
||||
return "pending"
|
||||
}
|
||||
|
||||
func toStringSlice(v interface{}) []string {
|
||||
if arr, ok := v.([]string); ok {
|
||||
return arr
|
||||
}
|
||||
if arr, ok := v.([]interface{}); ok {
|
||||
var result []string
|
||||
for _, item := range arr {
|
||||
if s, ok := item.(string); ok {
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isContradiction(a, b string) bool {
|
||||
// 简单启发式：重叠词 > 50% 但存在否定词差异
|
||||
wordsA := strings.Fields(strings.ToLower(a))
|
||||
wordsB := strings.Fields(strings.ToLower(b))
|
||||
setA := make(map[string]bool)
|
||||
for _, w := range wordsA {
|
||||
setA[w] = true
|
||||
}
|
||||
|
||||
overlap := 0
|
||||
negInA := containsNeg(wordsA)
|
||||
negInB := containsNeg(wordsB)
|
||||
|
||||
for _, w := range wordsB {
|
||||
if setA[w] {
|
||||
overlap++
|
||||
}
|
||||
}
|
||||
|
||||
totalOverlap := float64(overlap) / math.Max(float64(len(wordsA)), float64(len(wordsB)))
|
||||
return totalOverlap > 0.5 && negInA != negInB
|
||||
}
|
||||
|
||||
func containsNeg(words []string) bool {
|
||||
negs := []string{"not", "no", "don't", "doesn't", "false", "错误", "不是", "没有", "禁止", "不允许"}
|
||||
for _, w := range words {
|
||||
for _, n := range negs {
|
||||
if strings.Contains(w, n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ─── 遗忘策略 ────────────────────────────────────────────
|
||||
|
||||
type Forgetter struct {
|
||||
decayRate float64
|
||||
}
|
||||
|
||||
func NewForgetter() *Forgetter {
|
||||
return &Forgetter{decayRate: 0.015}
|
||||
}
|
||||
|
||||
// ShouldForget 判断记忆是否该被遗忘
|
||||
func (f *Forgetter) ShouldForget(lastAccessed time.Time, recallCount int, tier string) bool {
|
||||
if tier == "core" {
|
||||
return false // 核心记忆永不遗忘
|
||||
}
|
||||
days := time.Since(lastAccessed).Hours() / 24
|
||||
score := 1.0 - days*f.decayRate
|
||||
if score < 0.1 {
|
||||
score = 0.1
|
||||
}
|
||||
// recallCount > 0 减缓衰减
|
||||
score += float64(recallCount) * 0.05
|
||||
return score < 0.2
|
||||
}
|
||||
|
||||
// DecayScore 计算衰减分数
|
||||
func (f *Forgetter) DecayScore(lastAccessed time.Time, recallCount int) float64 {
|
||||
days := time.Since(lastAccessed).Hours() / 24
|
||||
score := 1.0 - days*f.decayRate
|
||||
if score < 0.1 {
|
||||
score = 0.1
|
||||
}
|
||||
score += float64(recallCount) * 0.05
|
||||
if score > 1.0 {
|
||||
score = 1.0
|
||||
}
|
||||
return math.Round(score*100) / 100
|
||||
}
|
||||
|
||||
// ─── 知识图谱 ────────────────────────────────────────────
|
||||
|
||||
type GraphStore struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func NewGraphStore(db *sql.DB) (*GraphStore, error) {
|
||||
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS graph_nodes (
|
||||
id TEXT PRIMARY KEY, name TEXT, type TEXT, namespace TEXT, created_at TEXT
|
||||
)`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS graph_edges (
|
||||
id TEXT PRIMARY KEY, source TEXT, target TEXT, relation TEXT, weight REAL, namespace TEXT, created_at TEXT,
|
||||
FOREIGN KEY (source) REFERENCES graph_nodes(id),
|
||||
FOREIGN KEY (target) REFERENCES graph_nodes(id)
|
||||
)`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &GraphStore{db: db}, nil
|
||||
}
|
||||
|
||||
// AddNode 添加节点
|
||||
func (gs *GraphStore) AddNode(id, name, nodeType, namespace string) error {
|
||||
_, err := gs.db.Exec(
|
||||
"INSERT OR REPLACE INTO graph_nodes (id, name, type, namespace, created_at) VALUES (?, ?, ?, ?, ?)",
|
||||
id, name, nodeType, namespace, time.Now().Format(time.RFC3339))
|
||||
return err
|
||||
}
|
||||
|
||||
// AddEdge 添加边
|
||||
func (gs *GraphStore) AddEdge(id, source, target, relation, namespace string, weight float64) error {
|
||||
_, err := gs.db.Exec(
|
||||
"INSERT OR REPLACE INTO graph_edges (id, source, target, relation, weight, namespace, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
id, source, target, relation, weight, namespace, time.Now().Format(time.RFC3339))
|
||||
return err
|
||||
}
|
||||
|
||||
// Navigate 多跳导航（双向 BFS）
|
||||
func (gs *GraphStore) Navigate(entity string, maxHops int, namespace string) ([]map[string]interface{}, error) {
|
||||
visited := map[string]bool{entity: true}
|
||||
queue := []string{entity}
|
||||
var paths []map[string]interface{}
|
||||
|
||||
for hop := 1; hop <= maxHops && len(queue) > 0; hop++ {
|
||||
var nextQueue []string
|
||||
for _, current := range queue {
|
||||
rows, err := gs.db.Query(
|
||||
`SELECT id, source, target, relation, weight FROM graph_edges
|
||||
WHERE (source = ? OR target = ?) AND namespace = ?`,
|
||||
current, current, namespace)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for rows.Next() {
|
||||
var id, source, target, relation string
|
||||
var weight float64
|
||||
rows.Scan(&id, &source, &target, &relation, &weight)
|
||||
|
||||
neighbor := target
|
||||
if current == target {
|
||||
neighbor = source
|
||||
}
|
||||
if visited[neighbor] {
|
||||
continue
|
||||
}
|
||||
visited[neighbor] = true
|
||||
nextQueue = append(nextQueue, neighbor)
|
||||
paths = append(paths, map[string]interface{}{
|
||||
"edge_id": id, "source": current, "target": neighbor,
|
||||
"relation": relation, "weight": weight, "hop": hop,
|
||||
})
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
queue = nextQueue
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
// Prune 修剪图谱（删除孤立节点、低权重边）
|
||||
func (gs *GraphStore) Prune(minWeight float64) error {
|
||||
_, err := gs.db.Exec("DELETE FROM graph_edges WHERE weight < ?", minWeight)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = gs.db.Exec(`DELETE FROM graph_nodes WHERE id NOT IN
|
||||
(SELECT DISTINCT source FROM graph_edges UNION SELECT DISTINCT target FROM graph_edges)`)
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,316 @@
|
|||
// 织忆 MemoryWeave — 自优化引擎
|
||||
package selfoptimize
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── 自优化仪表盘 ────────────────────────────────────────
|
||||
|
||||
type Dashboard struct {
|
||||
mu sync.RWMutex
|
||||
UsefulCount int `json:"useful_count"`
|
||||
NotUsefulCount int `json:"not_useful_count"`
|
||||
TotalRecalls int `json:"total_recalls"`
|
||||
HitCount int `json:"hit_count"`
|
||||
ClosedGaps int `json:"closed_gaps"`
|
||||
TotalGaps int `json:"total_gaps"`
|
||||
CascadeFixedTotal int `json:"cascade_fixed_total"`
|
||||
TotalFixes int `json:"total_fixes"`
|
||||
DeprecatedToday int `json:"deprecated_today"`
|
||||
DistillLossSum float64 `json:"distill_loss_sum"`
|
||||
DistillLossCount int `json:"distill_loss_count"`
|
||||
AutoResolvedConflicts int `json:"auto_resolved_conflicts"`
|
||||
TotalConflicts int `json:"total_conflicts"`
|
||||
}
|
||||
|
||||
var Dash = &Dashboard{}
|
||||
|
||||
// Metrics 返回 7 项核心指标
|
||||
func (d *Dashboard) Metrics() map[string]float64 {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
usefulRate := 0.0
|
||||
if d.UsefulCount+d.NotUsefulCount > 0 {
|
||||
usefulRate = float64(d.UsefulCount) / float64(d.UsefulCount+d.NotUsefulCount)
|
||||
}
|
||||
|
||||
hitRate := 0.0
|
||||
if d.TotalRecalls > 0 {
|
||||
hitRate = float64(d.HitCount) / float64(d.TotalRecalls)
|
||||
}
|
||||
|
||||
gapRate := 0.0
|
||||
if d.TotalGaps > 0 {
|
||||
gapRate = float64(d.ClosedGaps) / float64(d.TotalGaps)
|
||||
}
|
||||
|
||||
cascadeRate := 0.0
|
||||
if d.TotalFixes > 0 {
|
||||
cascadeRate = float64(d.CascadeFixedTotal) / float64(d.TotalFixes)
|
||||
}
|
||||
|
||||
avgLoss := 0.0
|
||||
if d.DistillLossCount > 0 {
|
||||
avgLoss = d.DistillLossSum / float64(d.DistillLossCount)
|
||||
}
|
||||
|
||||
autoRate := 0.0
|
||||
if d.TotalConflicts > 0 {
|
||||
autoRate = float64(d.AutoResolvedConflicts) / float64(d.TotalConflicts)
|
||||
}
|
||||
|
||||
return map[string]float64{
|
||||
"recall_usefulness_rate": math.Round(usefulRate*100) / 100,
|
||||
"recall_hit_rate": math.Round(hitRate*100) / 100,
|
||||
"gap_closure_rate": math.Round(gapRate*100) / 100,
|
||||
"cascade_fix_rate": math.Round(cascadeRate*100) / 100,
|
||||
"deprecated_per_day": float64(d.DeprecatedToday),
|
||||
"avg_distill_loss": math.Round(avgLoss*100) / 100,
|
||||
"auto_resolve_rate": math.Round(autoRate*100) / 100,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dashboard) RecordRecall(hit bool) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.TotalRecalls++
|
||||
if hit {
|
||||
d.HitCount++
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dashboard) RecordFeedback(useful bool) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if useful {
|
||||
d.UsefulCount++
|
||||
} else {
|
||||
d.NotUsefulCount++
|
||||
}
|
||||
}
|
||||
|
||||
// ─── 知识缺口检测 ────────────────────────────────────────
|
||||
|
||||
type GapType string
|
||||
|
||||
const (
|
||||
GapUnknown GapType = "A" // 真不知道
|
||||
GapSynonym GapType = "B" // 同义词不匹配
|
||||
GapRecallFailed GapType = "C" // 召回失败
|
||||
GapFragmented GapType = "D" // 碎片化
|
||||
)
|
||||
|
||||
type Gap struct {
|
||||
Topic string `json:"topic"`
|
||||
Type GapType `json:"type"`
|
||||
MissCount int `json:"miss_count"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Closed bool `json:"closed"`
|
||||
}
|
||||
|
||||
type GapDetector struct {
|
||||
mu sync.RWMutex
|
||||
gaps map[string]*Gap
|
||||
misses map[string]int
|
||||
}
|
||||
|
||||
func NewGapDetector() *GapDetector {
|
||||
return &GapDetector{
|
||||
gaps: make(map[string]*Gap),
|
||||
misses: make(map[string]int),
|
||||
}
|
||||
}
|
||||
|
||||
// RecordMiss 记录一次召回失败
|
||||
func (gd *GapDetector) RecordMiss(topic string) *Gap {
|
||||
gd.mu.Lock()
|
||||
defer gd.mu.Unlock()
|
||||
|
||||
gd.misses[topic]++
|
||||
if gd.misses[topic] >= 3 {
|
||||
if _, exists := gd.gaps[topic]; !exists {
|
||||
gap := &Gap{
|
||||
Topic: topic,
|
||||
Type: gd.classifyGap(topic),
|
||||
MissCount: gd.misses[topic],
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
gd.gaps[topic] = gap
|
||||
return gap
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (gd *GapDetector) classifyGap(topic string) GapType {
|
||||
// 简单启发式：大写缩写 → 同义词；中文 → 可能是真的不知道
|
||||
for _, r := range topic {
|
||||
if r >= 'A' && r <= 'Z' {
|
||||
return GapSynonym
|
||||
}
|
||||
}
|
||||
return GapUnknown
|
||||
}
|
||||
|
||||
func (gd *GapDetector) List() []*Gap {
|
||||
gd.mu.RLock()
|
||||
defer gd.mu.RUnlock()
|
||||
var result []*Gap
|
||||
for _, g := range gd.gaps {
|
||||
result = append(result, g)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (gd *GapDetector) Close(topic string) {
|
||||
gd.mu.Lock()
|
||||
defer gd.mu.Unlock()
|
||||
if g, ok := gd.gaps[topic]; ok {
|
||||
g.Closed = true
|
||||
}
|
||||
}
|
||||
|
||||
// ─── 因果追踪 ────────────────────────────────────────────
|
||||
|
||||
type TraceEntry struct {
|
||||
MemoryID string `json:"memory_id"`
|
||||
Version int `json:"version"`
|
||||
Content string `json:"content"`
|
||||
Source string `json:"source"` // muchen_oral / config_parse / agent_infer / llm_distill
|
||||
Trigger string `json:"trigger"` // 什么触发了这次修改
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type CausalTracker struct {
|
||||
mu sync.RWMutex
|
||||
entries map[string][]*TraceEntry // memory_id → version history
|
||||
deps map[string][]string // memory_id → depends_on[]
|
||||
}
|
||||
|
||||
func NewCausalTracker() *CausalTracker {
|
||||
return &CausalTracker{
|
||||
entries: make(map[string][]*TraceEntry),
|
||||
deps: make(map[string][]string),
|
||||
}
|
||||
}
|
||||
|
||||
// RecordVersion 记录版本变更
|
||||
func (ct *CausalTracker) RecordVersion(memoryID, content, source, trigger string) {
|
||||
ct.mu.Lock()
|
||||
defer ct.mu.Unlock()
|
||||
|
||||
entry := &TraceEntry{
|
||||
MemoryID: memoryID,
|
||||
Version: len(ct.entries[memoryID]) + 1,
|
||||
Content: content,
|
||||
Source: source,
|
||||
Trigger: trigger,
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
ct.entries[memoryID] = append(ct.entries[memoryID], entry)
|
||||
}
|
||||
|
||||
// AddDependency A depends_on B
|
||||
func (ct *CausalTracker) AddDependency(a, b string) {
|
||||
ct.mu.Lock()
|
||||
defer ct.mu.Unlock()
|
||||
ct.deps[a] = append(ct.deps[a], b)
|
||||
}
|
||||
|
||||
// GetAffected 当 memoryID 被修正时，返回所有依赖它的记忆
|
||||
func (ct *CausalTracker) GetAffected(memoryID string, visited map[string]bool) []string {
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
|
||||
if visited == nil {
|
||||
visited = make(map[string]bool)
|
||||
}
|
||||
if visited[memoryID] {
|
||||
return nil
|
||||
}
|
||||
visited[memoryID] = true
|
||||
|
||||
var affected []string
|
||||
for dependent, deps := range ct.deps {
|
||||
for _, d := range deps {
|
||||
if d == memoryID && !visited[dependent] {
|
||||
affected = append(affected, dependent)
|
||||
affected = append(affected, ct.GetAffected(dependent, visited)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return affected
|
||||
}
|
||||
|
||||
// SourceTrust 来源信任度
|
||||
func SourceTrust(source string) float64 {
|
||||
switch source {
|
||||
case "muchen_oral":
|
||||
return 1.0
|
||||
case "muchen_feishu":
|
||||
return 0.95
|
||||
case "config_parse":
|
||||
return 0.7
|
||||
case "agent_infer":
|
||||
return 0.5
|
||||
case "llm_distill":
|
||||
return 0.4
|
||||
default:
|
||||
return 0.3
|
||||
}
|
||||
}
|
||||
|
||||
// IsVolatile 判断一条记忆是否易变（频繁修正）
|
||||
func (ct *CausalTracker) IsVolatile(memoryID string) bool {
|
||||
ct.mu.RLock()
|
||||
defer ct.mu.RUnlock()
|
||||
return len(ct.entries[memoryID]) >= 3
|
||||
}
|
||||
|
||||
// ─── 记忆预取 ────────────────────────────────────────────
|
||||
|
||||
type PrefetchGraph struct {
|
||||
mu sync.RWMutex
|
||||
coOccurs map[string]map[string]int // A → {B: count}
|
||||
}
|
||||
|
||||
func NewPrefetchGraph() *PrefetchGraph {
|
||||
return &PrefetchGraph{coOccurs: make(map[string]map[string]int)}
|
||||
}
|
||||
|
||||
func (pg *PrefetchGraph) RecordCoAccess(a, b string) {
|
||||
pg.mu.Lock()
|
||||
defer pg.mu.Unlock()
|
||||
if pg.coOccurs[a] == nil {
|
||||
pg.coOccurs[a] = make(map[string]int)
|
||||
}
|
||||
pg.coOccurs[a][b]++
|
||||
}
|
||||
|
||||
// GetPrefetch 获取某个 query 的预取候选项
|
||||
func (pg *PrefetchGraph) GetPrefetch(query string) []string {
|
||||
pg.mu.RLock()
|
||||
defer pg.mu.RUnlock()
|
||||
|
||||
related := pg.coOccurs[query]
|
||||
if related == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var result []string
|
||||
for topic, count := range related {
|
||||
total := 0
|
||||
for _, c := range pg.coOccurs[query] {
|
||||
total += c
|
||||
}
|
||||
prob := float64(count) / float64(total)
|
||||
if prob > 0.6 {
|
||||
result = append(result, topic)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/xiaoxue/memoryweave/internal/models"
|
||||
)
|
||||
|
|
@ -159,7 +160,7 @@ func (c *LanceClient) SoftDelete(id, reason string) error {
|
|||
}
|
||||
|
||||
// Stats 返回各表记录数。
|
||||
func (c *LanceClient) Stats() (map[string]int, error) {
|
||||
func (c *LanceClient) Stats() (map[string]interface{}, error) {
|
||||
req, _ := http.NewRequest("GET", fmt.Sprintf("%s/v1/stats", c.baseURL), nil)
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
|
|
@ -167,9 +168,55 @@ func (c *LanceClient) Stats() (map[string]int, error) {
|
|||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var stats map[string]int
|
||||
var stats map[string]interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
|
||||
return nil, fmt.Errorf("decode stats: %w", err)
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// InsertEpisode 插入一条 episode 记录，返回 ID
|
||||
func (c *LanceClient) InsertEpisode(agentID, namespace, content, category string) (string, error) {
|
||||
id := fmt.Sprintf("ep_%d", time.Now().UnixNano())
|
||||
ep := models.EpisodeRecord{
|
||||
ID: id,
|
||||
AgentID: agentID,
|
||||
Namespace: namespace,
|
||||
Content: content,
|
||||
Category: category,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
return id, c.Insert("episodes", ep)
|
||||
}
|
||||
|
||||
// GetTopByQuality 按 quality_score 降序返回高质量记忆
|
||||
func (c *LanceClient) GetTopByQuality(agentID string, limit int) ([]models.MemoryRecord, error) {
|
||||
reqBody := map[string]interface{}{
|
||||
"top_k": limit,
|
||||
"filter": "is_deleted = false",
|
||||
"order": "quality_score DESC",
|
||||
}
|
||||
body, _ := json.Marshal(reqBody)
|
||||
req, _ := http.NewRequest("POST", fmt.Sprintf("%s/v1/table/memories/query", c.baseURL),
|
||||
bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var results []models.MemoryRecord
|
||||
if err := json.NewDecoder(resp.Body).Decode(&results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// InsertMemory 插入一条蒸馏后的记忆
|
||||
func (c *LanceClient) InsertMemory(m models.MemoryRecord) error {
|
||||
return c.Insert("memories", m)
|
||||
}
|
||||
|
||||
// LanceDB 类型别名，兼容路由层引用
|
||||
type LanceDB = LanceClient
|
||||
|
|
|
|||
Loading…
Reference in New Issue