病根:两个列表/导出端点一直用『零向量 ANN』冒充『列出全部』——零向量没有方向, ANN 退化成本存储序前 N 行,再按 ns 客户端过滤 ⇒ 计数完全失真 (实测 export hermes-main 报 1 条,真实 139+;shared 报 0,真实 29)。 修复: ① Rust lancedb_ops: scan_for_forgetting 加 namespace 下推(only_if),limit 作用于过滤后结果集 ② Go 接口 LanceDB 新增 Scan(limit, ns): - RustLanceDBClient → lancedb_scan RPC - SQLiteClient → SELECT ... WHERE is_deleted=0 [AND ns] ORDER BY created_at DESC LIMIT - MemLanceClient → 内存 map 过滤 + created_at 倒序 ③ 两个端点 ListMemories / ExportMemoriesMD 改用 aa.LanceDB.Scan(limit, ns) ④ 补 mock(zz_commit mockLance / write-buffer fakeLDB)+ 5 条新判据 (全表 3 / hermes-main 2 / shared 1 / limit 生效 / 不存在 ns 0)
544 lines
16 KiB
Go
544 lines
16 KiB
Go
// 织忆 MemoryWeave — 管理端点
|
||
package routes
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"log"
|
||
"net/http"
|
||
"os"
|
||
"os/exec"
|
||
"path/filepath"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/xiaoxue/memoryweave/internal/governance"
|
||
"github.com/xiaoxue/memoryweave/internal/storage"
|
||
)
|
||
|
||
type AdminAPI struct {
|
||
LanceDB storage.LanceDB
|
||
Forgetter *governance.Forgetter
|
||
GraphStore governance.GraphStore // E4.3: 图谱度参与遗忘决策
|
||
}
|
||
|
||
func NewAdminAPI(ldb storage.LanceDB, f *governance.Forgetter, gs governance.GraphStore) *AdminAPI {
|
||
return &AdminAPI{LanceDB: ldb, Forgetter: f, GraphStore: gs}
|
||
}
|
||
|
||
// DELETE /api/v1/distilled/{id}
|
||
func (aa *AdminAPI) DeleteDistilled(w http.ResponseWriter, r *http.Request) {
|
||
id := r.PathValue("id")
|
||
if id == "" {
|
||
respondError(w, 400, "id required")
|
||
return
|
||
}
|
||
if err := aa.LanceDB.SoftDelete(id, "manual_delete"); err != nil {
|
||
respondError(w, 500, "delete failed: "+err.Error())
|
||
return
|
||
}
|
||
respond(w, 200, map[string]string{"status": "deleted", "id": id})
|
||
}
|
||
|
||
// GET /api/v1/memory/{id}/versions
|
||
func (aa *AdminAPI) Versions(w http.ResponseWriter, r *http.Request) {
|
||
id := r.PathValue("id")
|
||
if id == "" {
|
||
respondError(w, 400, "id required")
|
||
return
|
||
}
|
||
versions, err := aa.LanceDB.GetVersionHistory(id)
|
||
if err != nil {
|
||
respondError(w, 500, "get versions failed: "+err.Error())
|
||
return
|
||
}
|
||
respond(w, 200, map[string]interface{}{
|
||
"memory_id": id, "versions": versions, "count": len(versions),
|
||
})
|
||
}
|
||
|
||
// POST /api/v1/admin/forget
|
||
func (aa *AdminAPI) Forget(w http.ResponseWriter, r *http.Request) {
|
||
results, err := aa.LanceDB.GetCandidatesForForgetting()
|
||
if err != nil {
|
||
respondError(w, 500, "list failed: "+err.Error())
|
||
return
|
||
}
|
||
|
||
forgotten := 0
|
||
for _, mem := range results {
|
||
lastAccess := parseTimeStr(mem["last_recalled_at"])
|
||
recallCnt := intVal(mem["recall_count"])
|
||
tier := strVal(mem["tier"])
|
||
content := strVal(mem["content"])
|
||
|
||
// 🔒 2026-09-07 修复(误删事故): episodes(原始对话)不参与遗忘
|
||
if cat := strVal(mem["category"]); cat == "episodes" {
|
||
continue
|
||
}
|
||
// 🔒 长内容保护: >200 字记忆不参与 auto 遗忘(有实质信息)
|
||
if len([]rune(content)) > 200 {
|
||
continue
|
||
}
|
||
// 2026-09-07 方案A 碎片快速道(与 server.go decay trigger 一致):
|
||
// <30 字且 >20 天未访问 → 直接遗忘(碎片 recall_count 可能虚高, 绕过保命)
|
||
if len([]rune(content)) < 30 && time.Since(lastAccess).Hours()/24 > 20 {
|
||
if err := aa.LanceDB.SoftDelete(strVal(mem["id"]), "auto_forget_fragment"); err != nil {
|
||
log.Printf("[softdelete] failed id: %v", err)
|
||
}
|
||
forgotten++
|
||
continue
|
||
}
|
||
|
||
// E4.3 修复: 从 content 提取实体(而非读 namespace),取图谱最大度
|
||
degree := extractTopEntityDegree(content, aa.GraphStore)
|
||
|
||
if aa.Forgetter.ShouldForget(lastAccess, recallCnt, tier, degree) {
|
||
if err := aa.LanceDB.SoftDelete(strVal(mem["id"]), "auto_forget"); err != nil {
|
||
log.Printf("[softdelete] failed id: %v", err)
|
||
}
|
||
forgotten++
|
||
}
|
||
}
|
||
respond(w, 200, map[string]interface{}{
|
||
"status": "ok", "scanned": len(results), "forgotten": forgotten,
|
||
})
|
||
}
|
||
|
||
// POST /api/v1/admin/backup
|
||
func (aa *AdminAPI) Backup(w http.ResponseWriter, r *http.Request) {
|
||
timestamp := time.Now().Format("20060102-150405")
|
||
backupDir := fmt.Sprintf("/home/muc/backups/memoryweave/%s", timestamp)
|
||
dataDir := "/var/lib/memoryweave"
|
||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||
respondError(w, 500, "mkdir failed: "+err.Error())
|
||
return
|
||
}
|
||
|
||
// 1. SQLite backup: graph.db
|
||
graphBackup := filepath.Join(backupDir, "graph.db")
|
||
if err := exec.Command("sqlite3", filepath.Join(dataDir, "graph.db"), ".backup "+graphBackup).Run(); err != nil {
|
||
respondError(w, 500, "graph.db backup failed: "+err.Error())
|
||
return
|
||
}
|
||
|
||
// 2. SQLite backup: memoryweave.db
|
||
mwBackup := filepath.Join(backupDir, "memoryweave.db")
|
||
if err := exec.Command("sqlite3", filepath.Join(dataDir, "memoryweave.db"), ".backup "+mwBackup).Run(); err != nil {
|
||
respondError(w, 500, "memoryweave.db backup failed: "+err.Error())
|
||
return
|
||
}
|
||
|
||
// 3. Redis BGSAVE (fire-and-forget)
|
||
exec.Command("redis-cli", "BGSAVE").Run()
|
||
|
||
// 4. Tar LanceDB directories (5 min timeout — 5.5GB takes ~100s)
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||
defer cancel()
|
||
if err := exec.CommandContext(ctx, "tar", "czf",
|
||
filepath.Join(backupDir, "lance-data.tar.gz"),
|
||
"-C", dataDir,
|
||
"episodes.lance", "memories.lance", "tombstones.lance",
|
||
).Run(); err != nil {
|
||
if ctx.Err() == context.DeadlineExceeded {
|
||
respondError(w, 500, "lance backup timed out after 5 minutes")
|
||
return
|
||
}
|
||
respondError(w, 500, "lance backup failed: "+err.Error())
|
||
return
|
||
}
|
||
|
||
respond(w, 200, map[string]string{
|
||
"status": "ok",
|
||
"path": backupDir,
|
||
"timestamp": timestamp,
|
||
})
|
||
}
|
||
|
||
// GET /api/v1/admin/backups — 列出可用备份
|
||
func (aa *AdminAPI) ListBackups(w http.ResponseWriter, r *http.Request) {
|
||
backupRoot := "/home/muc/backups/memoryweave"
|
||
entries, err := os.ReadDir(backupRoot)
|
||
if err != nil {
|
||
respondError(w, 500, "read dir failed: "+err.Error())
|
||
return
|
||
}
|
||
|
||
var backups []map[string]string
|
||
for _, e := range entries {
|
||
if !e.IsDir() {
|
||
continue
|
||
}
|
||
ts := e.Name()
|
||
// 读 mtime 作为备份时间
|
||
info, _ := e.Info()
|
||
modTime := info.ModTime().Format(time.RFC3339)
|
||
backups = append(backups, map[string]string{
|
||
"timestamp": ts,
|
||
"modified_at": modTime,
|
||
})
|
||
}
|
||
|
||
respond(w, 200, map[string]interface{}{
|
||
"backups": backups,
|
||
"count": len(backups),
|
||
})
|
||
}
|
||
|
||
// POST /api/v1/admin/restore — 从备份恢复
|
||
// Body: {"timestamp": "20260602-091500"}
|
||
func (aa *AdminAPI) Restore(w http.ResponseWriter, r *http.Request) {
|
||
var req struct {
|
||
Timestamp string `json:"timestamp"`
|
||
}
|
||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Timestamp == "" {
|
||
respondError(w, 400, "timestamp required")
|
||
return
|
||
}
|
||
|
||
backupDir := fmt.Sprintf("/home/muc/backups/memoryweave/%s", req.Timestamp)
|
||
if _, err := os.Stat(backupDir); os.IsNotExist(err) {
|
||
respondError(w, 404, "backup not found: "+req.Timestamp)
|
||
return
|
||
}
|
||
|
||
dataDir := "/var/lib/memoryweave"
|
||
|
||
// 1. 停止服务(先 Go 再 sidecar)
|
||
stop := func(svc string) error {
|
||
out, err := exec.Command("systemctl", "--user", "stop", svc).CombinedOutput()
|
||
if err != nil {
|
||
return fmt.Errorf("%s: %s", svc, string(out))
|
||
}
|
||
return nil
|
||
}
|
||
if err := stop("zhiyid.service"); err != nil {
|
||
respondError(w, 500, "stop zhiyid failed: "+err.Error())
|
||
return
|
||
}
|
||
if err := stop("zhiyi-sidecar.service"); err != nil {
|
||
respondError(w, 500, "stop sidecar failed: "+err.Error())
|
||
return
|
||
}
|
||
// 等进程退出
|
||
time.Sleep(2 * time.Second)
|
||
|
||
// 2. 清理旧数据(lances)
|
||
lanceTables := []string{"episodes.lance", "memories.lance", "tombstones.lance"}
|
||
for _, t := range lanceTables {
|
||
p := filepath.Join(dataDir, t)
|
||
os.RemoveAll(p + ".table")
|
||
os.RemoveAll(p)
|
||
}
|
||
|
||
// 3. 解压 LanceDB tar
|
||
tarPath := filepath.Join(backupDir, "lance-data.tar.gz")
|
||
if _, err := os.Stat(tarPath); err == nil {
|
||
cmd := exec.Command("tar", "xzf", tarPath, "-C", dataDir)
|
||
cmd.Dir = dataDir
|
||
if out, err := cmd.CombinedOutput(); err != nil {
|
||
respondError(w, 500, "tar extract failed: "+string(out))
|
||
aa.startServices()
|
||
return
|
||
}
|
||
}
|
||
|
||
// 4. 还原 SQLite
|
||
graphDst := filepath.Join(dataDir, "graph.db")
|
||
mwDst := filepath.Join(dataDir, "memoryweave.db")
|
||
if err := exec.Command("sqlite3", graphDst, ".restore "+filepath.Join(backupDir, "graph.db")).Run(); err != nil {
|
||
respondError(w, 500, "graph.db restore failed: "+err.Error())
|
||
aa.startServices()
|
||
return
|
||
}
|
||
if err := exec.Command("sqlite3", mwDst, ".restore "+filepath.Join(backupDir, "memoryweave.db")).Run(); err != nil {
|
||
respondError(w, 500, "memoryweave.db restore failed: "+err.Error())
|
||
aa.startServices()
|
||
return
|
||
}
|
||
|
||
// 5. 重启服务(先 sidecar 再 Go)
|
||
aa.startServices()
|
||
|
||
respond(w, 200, map[string]string{
|
||
"status": "ok",
|
||
"restored": req.Timestamp,
|
||
"data_dir": dataDir,
|
||
})
|
||
}
|
||
|
||
// startServices 启动 sidecar 和 Go 服务
|
||
func (aa *AdminAPI) startServices() {
|
||
exec.Command("systemctl", "--user", "start", "zhiyi-sidecar.service").Run()
|
||
time.Sleep(1 * time.Second)
|
||
exec.Command("systemctl", "--user", "start", "zhiyid.service").Run()
|
||
// 等待 API 就绪
|
||
for i := 0; i < 10; i++ {
|
||
time.Sleep(1 * time.Second)
|
||
if resp, err := http.Get("http://localhost:7821/api/v1/stats"); err == nil {
|
||
resp.Body.Close()
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
// GET /api/v1/admin/audit
|
||
func (aa *AdminAPI) Audit(w http.ResponseWriter, r *http.Request) {
|
||
limit := 100
|
||
logs, err := aa.LanceDB.GetAuditLog(limit)
|
||
if err != nil {
|
||
respondError(w, 500, "audit failed: "+err.Error())
|
||
return
|
||
}
|
||
respond(w, 200, map[string]interface{}{"audit_logs": logs, "count": len(logs)})
|
||
}
|
||
|
||
// ─── 辅助函数 ───────────────────────────────────────────────────────────────
|
||
|
||
func parseTimeStr(s interface{}) time.Time {
|
||
if s == nil {
|
||
return time.Time{}
|
||
}
|
||
switch v := s.(type) {
|
||
case string:
|
||
t, _ := time.Parse(time.RFC3339, v)
|
||
return t
|
||
case time.Time:
|
||
return v
|
||
}
|
||
return time.Time{}
|
||
}
|
||
|
||
func intVal(v interface{}) int {
|
||
switch n := v.(type) {
|
||
case int:
|
||
return n
|
||
case int32:
|
||
return int(n)
|
||
case int64:
|
||
return int(n)
|
||
case float64:
|
||
return int(n)
|
||
case json.Number:
|
||
i, _ := n.Int64()
|
||
return int(i)
|
||
}
|
||
return 0
|
||
}
|
||
|
||
func strVal(v interface{}) string {
|
||
if v == nil {
|
||
return ""
|
||
}
|
||
switch s := v.(type) {
|
||
case string:
|
||
return s
|
||
case json.Number:
|
||
return s.String()
|
||
}
|
||
return fmt.Sprintf("%v", v)
|
||
}
|
||
|
||
// ─── E4.3: 图谱度提取 ──────────────────────────────────────────────────────
|
||
|
||
// extractTopEntityDegree 从 content 提取实体,返回图中度数最高的实体度数
|
||
// 复用 distill/engine.go 的启发式逻辑(大写单词、中文实体、技术标记)
|
||
func extractTopEntityDegree(content string, gs governance.GraphStore) int {
|
||
if content == "" || gs == nil {
|
||
return 0
|
||
}
|
||
seen := make(map[string]bool)
|
||
var candidates []string
|
||
|
||
words := strings.Fields(content)
|
||
for _, w := range words {
|
||
w = strings.Trim(w, ",.;:!?,。;:!?、\"'()()[]【】")
|
||
if len(w) < 2 {
|
||
continue
|
||
}
|
||
// 大写字母开头的英文词(Hermes, ComfyUI, Redis 等)
|
||
runes := []rune(w)
|
||
if len(runes) >= 2 && runes[0] >= 'A' && runes[0] <= 'Z' {
|
||
normalized := strings.ToLower(w)
|
||
if !seen[normalized] && !isStopWord(normalized) {
|
||
seen[normalized] = true
|
||
candidates = append(candidates, w)
|
||
}
|
||
}
|
||
// 中文实体(2-20 个纯中文字符)
|
||
cleanChinese := stripNonChinese(w)
|
||
if len(cleanChinese) >= 2 && len(cleanChinese) <= 20 {
|
||
if !seen[cleanChinese] {
|
||
seen[cleanChinese] = true
|
||
candidates = append(candidates, cleanChinese)
|
||
}
|
||
}
|
||
// 技术标记(数字+字母组合或纯数字)
|
||
if isTechToken(w) || isAllDigits(w) {
|
||
if !seen[w] {
|
||
seen[w] = true
|
||
candidates = append(candidates, w)
|
||
}
|
||
}
|
||
}
|
||
|
||
maxDegree := 0
|
||
for _, entity := range candidates {
|
||
d := gs.GetEntityDegree(entity)
|
||
if d > maxDegree {
|
||
maxDegree = d
|
||
}
|
||
}
|
||
return maxDegree
|
||
}
|
||
|
||
// isStopWord 停用词表(与 distill/engine.go 保持一致)
|
||
func isStopWord(w string) bool {
|
||
stops := []string{
|
||
"the", "and", "for", "are", "but", "not", "you", "all", "can", "had",
|
||
"her", "was", "one", "our", "out", "this", "that", "with", "from",
|
||
"your", "what", "when", "where", "which", "their", "will", "would",
|
||
"there", "could", "other", "into", "just", "has", "have", "were",
|
||
"they", "been", "more", "than",
|
||
}
|
||
for _, s := range stops {
|
||
if w == s {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// stripNonChinese 提取纯中文字符串
|
||
func stripNonChinese(s string) string {
|
||
var result []rune
|
||
for _, r := range s {
|
||
if r >= 0x4E00 && r <= 0x9FFF {
|
||
result = append(result, r)
|
||
}
|
||
}
|
||
return string(result)
|
||
}
|
||
|
||
// isTechToken 判断是否为技术标记(数字+字母混合)
|
||
func isTechToken(s string) bool {
|
||
hasDigit := false
|
||
hasLetter := false
|
||
for _, r := range s {
|
||
if r >= '0' && r <= '9' {
|
||
hasDigit = true
|
||
}
|
||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') {
|
||
hasLetter = true
|
||
}
|
||
}
|
||
return hasDigit && hasLetter
|
||
}
|
||
|
||
// isAllDigits 判断是否全为数字
|
||
func isAllDigits(s string) bool {
|
||
if len(s) == 0 {
|
||
return false
|
||
}
|
||
for _, r := range s {
|
||
if r < '0' || r > '9' {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// GET /api/v1/memories — 批量列出记忆(按时间倒序,不含向量)
|
||
// 2026-09-21:改用全表扫描(aa.LanceDB.Scan)替代「零向量 ANN 冒充列表」。
|
||
// 修前零向量无方向 ⇒ 只拿到存储序前 N 行再按 ns 过滤 ⇒ 列表/计数失真。
|
||
func (aa *AdminAPI) ListMemories(w http.ResponseWriter, r *http.Request) {
|
||
limit := 100
|
||
ns := r.URL.Query().Get("namespace")
|
||
if l := r.URL.Query().Get("limit"); l != "" {
|
||
if v, err := fmt.Sscanf(l, "%d", &limit); err != nil || v != 1 || limit < 1 {
|
||
limit = 100
|
||
}
|
||
if limit > 1000 {
|
||
limit = 1000
|
||
}
|
||
}
|
||
results, err := aa.LanceDB.Scan(limit, ns)
|
||
if err != nil {
|
||
respondError(w, 500, "list memories: "+err.Error())
|
||
return
|
||
}
|
||
// 去掉向量字段,减少响应体积
|
||
type memoryItem struct {
|
||
ID string `json:"id"`
|
||
Content string `json:"content"`
|
||
Category string `json:"category"`
|
||
Namespace string `json:"namespace,omitempty"`
|
||
Importance float64 `json:"importance"`
|
||
CreatedAt int64 `json:"created_at"`
|
||
}
|
||
items := make([]memoryItem, 0, len(results))
|
||
for _, m := range results {
|
||
items = append(items, memoryItem{
|
||
ID: m.ID, Content: m.Content, Category: m.Category,
|
||
Namespace: m.Namespace, Importance: m.Importance,
|
||
CreatedAt: m.CreatedAt.Unix(),
|
||
})
|
||
}
|
||
respond(w, 200, map[string]interface{}{
|
||
"memories": items, "count": len(items), "limit": limit,
|
||
})
|
||
}
|
||
|
||
// ExportMemoriesMD 导出记忆为 Markdown(P5:EverOS 式 md 真相层)
|
||
// GET /api/v1/memories/export?namespace=hermes-main&limit=1000&format=md
|
||
// 输出人可读的 Markdown 文档:记忆可迁移、可备份、可人工审查
|
||
func (aa *AdminAPI) ExportMemoriesMD(w http.ResponseWriter, r *http.Request) {
|
||
ns := r.URL.Query().Get("namespace")
|
||
limit := 1000
|
||
if l := r.URL.Query().Get("limit"); l != "" {
|
||
if v, err := fmt.Sscanf(l, "%d", &limit); err != nil || v != 1 || limit < 1 {
|
||
limit = 1000
|
||
}
|
||
if limit > 5000 {
|
||
limit = 5000
|
||
}
|
||
}
|
||
// 2026-09-21:改用全表扫描(aa.LanceDB.Scan)替代「零向量 ANN 冒充列表」。
|
||
results, err := aa.LanceDB.Scan(limit, ns)
|
||
if err != nil {
|
||
respondError(w, 500, "export memories: "+err.Error())
|
||
return
|
||
}
|
||
|
||
var sb strings.Builder
|
||
sb.WriteString("# 织忆记忆导出\n\n")
|
||
sb.WriteString(fmt.Sprintf("> 导出时间: %s\n", time.Now().Format("2006-01-02 15:04:05")))
|
||
sb.WriteString(fmt.Sprintf("> 命名空间: %s | 条数: %d\n\n---\n\n", orDefault(ns, "all"), len(results)))
|
||
|
||
for i, m := range results {
|
||
sb.WriteString(fmt.Sprintf("## M%d — %s\n\n", i+1, m.ID))
|
||
sb.WriteString(fmt.Sprintf("- **分类**: %s\n", orDefault(m.Category, "unknown")))
|
||
if m.Namespace != "" {
|
||
sb.WriteString(fmt.Sprintf("- **命名空间**: %s\n", m.Namespace))
|
||
}
|
||
sb.WriteString(fmt.Sprintf("- **重要性**: %.2f\n", m.Importance))
|
||
sb.WriteString(fmt.Sprintf("- **时间**: %s\n", m.CreatedAt.Format("2006-01-02 15:04:05")))
|
||
sb.WriteString("\n### 内容\n\n")
|
||
sb.WriteString(m.Content)
|
||
sb.WriteString("\n\n---\n\n")
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
|
||
w.Header().Set("Content-Disposition", "attachment; filename=zhiyi-memories-"+time.Now().Format("20060102")+".md")
|
||
w.WriteHeader(200)
|
||
w.Write([]byte(sb.String()))
|
||
}
|
||
|
||
func orDefault(s, def string) string {
|
||
if s == "" {
|
||
return def
|
||
}
|
||
return s
|
||
}
|