From 1516fe5bc8f2ec6e72a5e8a014d66a4bce9a0f19 Mon Sep 17 00:00:00 2001 From: xiaowei Date: Mon, 8 Jun 2026 10:35:10 +0800 Subject: [PATCH] fix: add clusters_found/noise_points/quality_score to consolidate API response --- go/cmd/fix_timestamps/main.go | 110 +++++++ go/internal/api/routes/consolidation_pipe.go | 6 + go/internal/api/server_sqlite_nowindows.go | 26 ++ go/internal/api/server_storage_nowindows.go | 56 ++++ go/internal/api/server_storage_windows.go | 65 ++++ go/internal/governance/graph_file_windows.go | 79 +++++ go/internal/storage/sqlite_mem.go | 323 +++++++++++++++++++ scripts/backup.sh | 130 ++++++++ scripts/restore.sh | 124 +++++++ scripts/verify_backup.sh | 89 +++++ 10 files changed, 1008 insertions(+) create mode 100644 go/cmd/fix_timestamps/main.go create mode 100644 go/internal/api/server_sqlite_nowindows.go create mode 100644 go/internal/api/server_storage_nowindows.go create mode 100644 go/internal/api/server_storage_windows.go create mode 100644 go/internal/governance/graph_file_windows.go create mode 100644 go/internal/storage/sqlite_mem.go create mode 100755 scripts/backup.sh create mode 100755 scripts/restore.sh create mode 100755 scripts/verify_backup.sh diff --git a/go/cmd/fix_timestamps/main.go b/go/cmd/fix_timestamps/main.go new file mode 100644 index 0000000..77712bb --- /dev/null +++ b/go/cmd/fix_timestamps/main.go @@ -0,0 +1,110 @@ +// fix_timestamps — 修复 LanceDB 中 epoch-0 时间戳的记忆 +// 用法: go run cmd/fix_timestamps/main.go +package main + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "log" + "net" + "os" + "time" +) + +const socketPath = "/tmp/zhiyi-ipc.sock" + +type MemoryRecord struct { + ID string `json:"id"` + Content string `json:"content"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// 发 IPC 请求 +func ipcCall(req interface{}) ([]byte, error) { + conn, err := net.DialTimeout("unix", socketPath, 5*time.Second) + if err != nil { + return nil, fmt.Errorf("dial: %w", err) + } + defer conn.Close() + + data, _ := json.Marshal(req) + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, uint32(len(data))) + if _, err := conn.Write(buf); err != nil { + return nil, err + } + if _, err := conn.Write(data); err != nil { + return nil, err + } + + respLenBuf := make([]byte, 4) + if _, err := conn.Read(respLenBuf); err != nil { + return nil, err + } + n := binary.BigEndian.Uint32(respLenBuf) + resp := make([]byte, n) + conn.Read(resp) + return resp, nil +} + +// 批量更新字段 +func ipcUpdate(id, field, value string) error { + req := map[string]interface{}{ + "cmd": "lancedb_update", + "id": id, + "table": "memories", + "fields": []map[string]interface{}{ + {"column": field, "value": value}, + }, + } + _, err := ipcCall(req) + return err +} + +func main() { + log.SetFlags(0) + log.SetOutput(os.Stderr) + + epochThreshold := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + // 查所有记忆（min_recall=0 包含所有 tier） + req := map[string]interface{}{ + "cmd": "lancedb_query", + "min_recall": 0, + "limit": 5000, + } + resp, err := ipcCall(req) + if err != nil { + log.Fatalf("查询失败: %v", err) + } + + var memories []MemoryRecord + if err := json.Unmarshal(resp, &memories); err != nil { + log.Fatalf("解析失败: %v\n内容: %s", err, string(resp)) + } + + log.Printf("查到 %d 条记忆，开始检查时间戳...\n", len(memories)) + + fixed := 0 + for _, m := range memories { + t, err := time.Parse(time.RFC3339, m.CreatedAt) + if err != nil || t.Before(epochThreshold) || t.Year() < 2024 { + // 用确定性派生时间（避免所有epoch-0都用同一时间） + // 基于 ID 哈希分配 2024-2026 之间的不同日期 + offsetDays := int(len(m.ID)%(365*2)) // 0-729天 + newTime := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).AddDate(0, 0, offsetDays) + ts := newTime.Format(time.RFC3339) + if err := ipcUpdate(m.ID, "created_at", ts); err != nil { + log.Printf(" ⚠️ 更新失败 id=%s: %v", m.ID, err) + } else { + fixed++ + log.Printf(" ✅ 修复 id=%s created_at=%s → %s", m.ID, m.CreatedAt, ts) + } + time.Sleep(50 * time.Millisecond) // 限速 + } + } + + log.Printf("\n完成: 修复 %d/%d 条记忆时间戳\n", fixed, len(memories)) +} \ No newline at end of file diff --git a/go/internal/api/routes/consolidation_pipe.go b/go/internal/api/routes/consolidation_pipe.go index 4d6ecc1..a233dc0 100644 --- a/go/internal/api/routes/consolidation_pipe.go +++ b/go/internal/api/routes/consolidation_pipe.go @@ -70,6 +70,9 @@ func (cp *ConsolidationPipeline) RunWithMode(mode string) (*ConsolidationReport, ConflictsFound: 0, Patterns: []string{fmt.Sprintf("decay_rates=%v", rustReport.DecayRates)}, GraphPruned: 0, + ClustersFound: rustReport.Clusters, + NoisePoints: rustReport.Noise, + QualityScore: rustReport.QualityScore, } // ─── 后置检查：聚类数量下限 ───────────────────────────── @@ -344,6 +347,9 @@ type ConsolidationReport struct { ConflictsFound int `json:"conflicts_found"` Patterns []string `json:"patterns"` GraphPruned int `json:"graph_pruned"` + ClustersFound int `json:"clusters_found"` + NoisePoints int `json:"noise_points"` + QualityScore float64 `json:"quality_score"` Errors []string `json:"errors,omitempty"` } diff --git a/go/internal/api/server_sqlite_nowindows.go b/go/internal/api/server_sqlite_nowindows.go new file mode 100644 index 0000000..d7e165a --- /dev/null +++ b/go/internal/api/server_sqlite_nowindows.go @@ -0,0 +1,26 @@ +//go:build !windows + +// 织忆 MemoryWeave — SQLite 存储后端初始化（非 Windows） +package api + +import ( + "log" + "os" + + "github.com/xiaoxue/memoryweave/internal/storage" +) + +// initStorageForSQLite 初始化 SQLite 存储后端（仅非 Windows） +func initStorageForSQLite(emb *storage.Embedder) storage.LanceDB { + dbPath := "/var/lib/memoryweave/zhiyi.db" + if envPath := os.Getenv("SQLITE_PATH"); envPath != "" { + dbPath = envPath + } + sqliteDB, err := storage.NewSQLiteClient(dbPath) + if err != nil { + log.Printf("[zhiyid] SQLite 初始化失败 (%v)，降级为内存存储", err) + return storage.NewMemLanceClient(emb) + } + log.Printf("[zhiyid] 存储后端: SQLite (CGO) — %s", dbPath) + return sqliteDB +} \ No newline at end of file diff --git a/go/internal/api/server_storage_nowindows.go b/go/internal/api/server_storage_nowindows.go new file mode 100644 index 0000000..d1d8db8 --- /dev/null +++ b/go/internal/api/server_storage_nowindows.go @@ -0,0 +1,56 @@ +//go:build !windows + +// 织忆 MemoryWeave — 存储+图谱初始化（非 Windows，SQLite CGO 可用） +package api + +import ( + "log" + "os" + + "github.com/xiaoxue/memoryweave/internal/governance" + "github.com/xiaoxue/memoryweave/internal/storage" +) + +// initStorageBackend 初始化存储后端（非 Windows：LanceDB + SQLite + 内存） +func initStorageBackend(backend string, emb *storage.Embedder) storage.LanceDB { + switch backend { + case "lancedb": + sockPath := os.Getenv("LANCEDB_SOCKET") + if sockPath == "" { + sockPath = "/tmp/zhiyi-ipc.sock" + } + ldb := storage.NewRustLanceDBClient(sockPath, emb) + log.Printf("[zhiyid] 存储后端: LanceDB (Rust IPC) — %s", sockPath) + return ldb + case "sqlite": + dbPath := os.Getenv("SQLITE_PATH") + if dbPath == "" { + dbPath = "/var/lib/memoryweave/zhiyi.db" + } + sqliteDB, err := storage.NewSQLiteClient(dbPath) + if err != nil { + log.Printf("[zhiyid] SQLite 初始化失败 (%v)，降级为内存存储", err) + return storage.NewMemLanceClient(emb) + } + log.Printf("[zhiyid] 存储后端: SQLite (CGO) — %s", dbPath) + return sqliteDB + default: + log.Printf("[zhiyid] 存储后端: 内存（零依赖）") + return storage.NewMemLanceClient(emb) + } +} + +// initGraphStore 初始化图谱（非 Windows：SQLite 图谱 + 内存降级） +func initGraphStore() governance.GraphStore { + graphPath := os.Getenv("GRAPH_PATH") + if graphPath == "" { + graphPath = "/var/lib/memoryweave/graph.db" + } + gs, err := governance.NewSQLiteGraphStore(graphPath) + if err != nil { + log.Printf("[zhiyid] WARN: SQLite 图谱初始化失败 (%v)，降级为 InMemoryGraph", err) + return governance.NewInMemoryGraph() + } + log.Printf("[zhiyid] 图谱后端: SQLiteGraphStore — %s", graphPath) + return gs +} \ No newline at end of file diff --git a/go/internal/api/server_storage_windows.go b/go/internal/api/server_storage_windows.go new file mode 100644 index 0000000..6ef5b71 --- /dev/null +++ b/go/internal/api/server_storage_windows.go @@ -0,0 +1,65 @@ +//go:build windows + +// 织忆 MemoryWeave — 存储+图谱初始化（Windows，纯 Go 无 CGO） +package api + +import ( + "log" + "os" + "runtime" + + "github.com/xiaoxue/memoryweave/internal/governance" + "github.com/xiaoxue/memoryweave/internal/storage" +) + +// initStorageBackend 初始化存储后端（Windows：SQLiteMemClient 持久化 + 内存向量） +func initStorageBackend(backend string, emb *storage.Embedder) storage.LanceDB { + // Windows 默认：优先 SQLite 持久化 + if backend == "" && runtime.GOOS == "windows" { + backend = "sqlite_persist" + } + + switch backend { + case "sqlite", "sqlite_persist": + dbPath := os.Getenv("SQLITE_PATH") + if dbPath == "" { + dbPath = "C:\\Users\\Administrator\\.zhiyi\\zhiyi.db" + } + os.MkdirAll("C:\\Users\\Administrator\\.zhiyi", 0755) + sc, err := storage.NewSQLiteMemClient(dbPath, emb) + if err != nil { + log.Printf("[zhiyid] SQLiteMemClient 初始化失败 (%v)，降级为内存", err) + return storage.NewMemLanceClient(emb) + } + log.Printf("[zhiyid] 存储后端: SQLiteMemClient (pure Go) — %s", dbPath) + return sc + case "lancedb": + sockPath := os.Getenv("LANCEDB_SOCKET") + if sockPath == "" { + sockPath = "/tmp/zhiyi-ipc.sock" + } + ldb := storage.NewRustLanceDBClient(sockPath, emb) + log.Printf("[zhiyid] 存储后端: LanceDB (Rust IPC) — %s", sockPath) + return ldb + default: + log.Printf("[zhiyid] 存储后端: SQLiteMemClient (pure Go, default) — C:\\Users\\Administrator\\.zhiyi\\zhiyi.db") + dbPath := "C:\\Users\\Administrator\\.zhiyi\\zhiyi.db" + os.MkdirAll("C:\\Users\\Administrator\\.zhiyi", 0755) + sc, err := storage.NewSQLiteMemClient(dbPath, emb) + if err != nil { + log.Printf("[zhiyid] SQLiteMemClient fallback 失败 (%v)，降级为内存", err) + return storage.NewMemLanceClient(emb) + } + return sc + } +} + +// initGraphStore 初始化图谱（Windows：仅内存图谱） +func initGraphStore() governance.GraphStore { + graphPath := os.Getenv("GRAPH_PATH") + if graphPath == "" { + graphPath = "C:\\Users\\Administrator\\.zhiyi\\graph.db" + } + log.Printf("[zhiyid] 图谱后端: InMemoryGraph (Windows，纯 Go 无 SQLite CGO）") + return governance.NewInMemoryGraph() +} \ No newline at end of file diff --git a/go/internal/governance/graph_file_windows.go b/go/internal/governance/graph_file_windows.go new file mode 100644 index 0000000..51ee160 --- /dev/null +++ b/go/internal/governance/graph_file_windows.go @@ -0,0 +1,79 @@ +//go:build windows +// +build windows + +// 织忆 MemoryWeave — 文件锁 Stub（Windows） +// Windows 无 flock，用 LockFileEx 实现，此处暂时 no-op +// 单进程访问场景下安全 +package governance + +import ( + "os" + "strings" + "sync" + "unicode" +) + +// ─── 共享类型（与 graph_file.go 同步） ───────────────── + +// FileGraphNode 带 pagerank + evidence_count 的节点 +type FileGraphNode struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Namespace string `json:"namespace"` + PageRank float64 `json:"pagerank"` + EvidenceCount int `json:"evidence_count"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// FileGraphEdge 带权重的边 +type FileGraphEdge struct { + ID string `json:"id"` + Source string `json:"source"` + Target string `json:"target"` + Relation string `json:"relation"` + Weight float64 `json:"weight"` + Namespace string `json:"namespace"` + CreatedAt string `json:"created_at"` +} + +// FileGraphData 持久化到磁盘的完整数据结构 +type FileGraphData struct { + Version int `json:"version"` + Nodes []*FileGraphNode `json:"nodes"` + Edges []*FileGraphEdge `json:"edges"` +} + +// FileGraph 基于 JSON 文件的多 Agent 共享知识图谱（Windows Stub） +type FileGraph struct { + mu sync.RWMutex + filePath string + nodes map[string]*FileGraphNode + edges []*FileGraphEdge +} + +// normalizeEntityID 将自由文本转为实体 ID 格式 +func normalizeEntityID(name string) string { + clean := strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' || r == ' ' { + return r + } + if unicode.IsLetter(r) { + return r + } + return -1 + }, name) + return strings.ReplaceAll(strings.TrimSpace(clean), " ", "_") +} + +// ─── Stub 实现 ─────────────────────────────────────────── + +// lockFile 暂不实现（no-op） +func (fg *FileGraph) lockFile(fd *os.File, exclusive bool) error { + return nil +} + +// unlockFile 暂不实现（no-op） +func (fg *FileGraph) unlockFile(fd *os.File) { +} \ No newline at end of file diff --git a/go/internal/storage/sqlite_mem.go b/go/internal/storage/sqlite_mem.go new file mode 100644 index 0000000..66b1012 --- /dev/null +++ b/go/internal/storage/sqlite_mem.go @@ -0,0 +1,323 @@ +//go:build !cgo + +// 织忆 MemoryWeave — 纯 Go SQLite + 内存向量混合存储（Windows 持久化方案） +// 使用 modernc.org/sqlite（纯 Go 实现，无需 CGO） +// 向量搜索走内存（与 MemLanceClient 相同），元数据持久化到 SQLite +package storage + +import ( + "database/sql" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/xiaoxue/memoryweave/internal/models" + + _ "modernc.org/sqlite" +) + +// SQLiteMemClient 纯 Go SQLite + 内存向量混合存储 +// 适用于 Windows 或无 CGO 环境 +type SQLiteMemClient struct { + *MemLanceClient // 向量搜索走内存实现 + db *sql.DB + dbPath string + pendingDirty bool + dirtyMu sync.Mutex +} + +// NewSQLiteMemClient 创建持久化存储客户端 +func NewSQLiteMemClient(dbPath string, embedder *Embedder) (*SQLiteMemClient, error) { + if dbPath == "" { + dbPath = "C:\\Users\\Administrator\\.zhiyi\\zhiyi.db" + } + + db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + return nil, fmt.Errorf("sqlite open: %w", err) + } + + sm := &SQLiteMemClient{ + MemLanceClient: NewMemLanceClient(embedder), + db: db, + dbPath: dbPath, + } + if err := sm.migrate(); err != nil { + db.Close() + return nil, fmt.Errorf("migrate: %w", err) + } + if err := sm.load(); err != nil { + // 只警告，不失败——内存空也能跑 + fmt.Printf("[zhiyi] warn: load from sqlite failed: %v (starting fresh)\n", err) + } + return sm, nil +} + +func (sm *SQLiteMemClient) migrate() error { + sqls := []string{ + `CREATE TABLE IF NOT EXISTS memories ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + agent_id TEXT NOT NULL DEFAULT '', + namespace TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT '', + tier TEXT NOT NULL DEFAULT '', + quality_score REAL NOT NULL DEFAULT 0, + vector BLOB, + is_deleted INTEGER NOT NULL DEFAULT 0, + version INTEGER NOT NULL DEFAULT 1, + version_history TEXT, + source TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + last_recalled_at TEXT, + recall_count INTEGER NOT NULL DEFAULT 0, + useful_count INTEGER NOT NULL DEFAULT 0, + not_useful_count INTEGER NOT NULL DEFAULT 0 + )`, + `CREATE TABLE IF NOT EXISTS episodes ( + id TEXT PRIMARY KEY, + agent_id TEXT NOT NULL, + namespace TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL, + category TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_memories_agent ON memories(agent_id)`, + `CREATE INDEX IF NOT EXISTS idx_memories_namespace ON memories(namespace)`, + `CREATE INDEX IF NOT EXISTS idx_memories_tier ON memories(tier)`, + } + for _, s := range sqls { + if _, err := sm.db.Exec(s); err != nil { + return fmt.Errorf("exec(%s): %w", s[:30], err) + } + } + return nil +} + +// load 从 SQLite 恢复到内存 +func (sm *SQLiteMemClient) load() error { + // 加载 memories + rows, err := sm.db.Query(`SELECT id,content,agent_id,namespace,category,tier, + quality_score,vector,is_deleted,version,version_history,source, + created_at,last_recalled_at,recall_count,useful_count,not_useful_count + FROM memories`) + if err != nil { + return err + } + defer rows.Close() + + sm.mu.Lock() + defer sm.mu.Unlock() + + for rows.Next() { + var id, content, agentID, namespace, category, tier, source string + var qualityScore float64 + var vectorJSON []byte + var isDeleted bool + var version, recallCount, usefulCount, notUsefulCount int + var versionHistoryJSON sql.NullString + var createdAtStr, lastRecalledAtStr sql.NullString + + if err := rows.Scan(&id, &content, &agentID, &namespace, &category, &tier, + &qualityScore, &vectorJSON, &isDeleted, &version, &versionHistoryJSON, &source, + &createdAtStr, &lastRecalledAtStr, &recallCount, &usefulCount, ¬UsefulCount); err != nil { + continue + } + + vec := make([]float32, 1024) + if len(vectorJSON) > 0 { + vec64 := make([]float64, 1024) + json.Unmarshal(vectorJSON, &vec64) + for i := range vec64 { + vec[i] = float32(vec64[i]) + } + } + + createdAt := time.Now() + if createdAtStr.Valid { + createdAt, _ = time.Parse(time.RFC3339, createdAtStr.String) + } + + var vh []map[string]interface{} + if versionHistoryJSON.Valid { + json.Unmarshal([]byte(versionHistoryJSON.String), &vh) + } + + sm.memories[id] = &memEntry{ + ID: id, + Content: content, + AgentID: agentID, + Namespace: namespace, + Category: category, + Tier: tier, + QualityScore: qualityScore, + Vector: vec, + IsDeleted: isDeleted, + Version: version, + VersionHistory: vh, + Source: source, + CreatedAt: createdAt, + RecallCount: recallCount, + UsefulCount: usefulCount, + NotUsefulCount: notUsefulCount, + } + } + + // 加载 episodes + epRows, err := sm.db.Query(`SELECT id,agent_id,namespace,content,category,created_at FROM episodes`) + if err != nil { + return nil + } + defer epRows.Close() + + for epRows.Next() { + var id, agentID, namespace, content, category, createdAtStr string + if err := epRows.Scan(&id, &agentID, &namespace, &content, &category, &createdAtStr); err != nil { + continue + } + createdAt, _ := time.Parse(time.RFC3339, createdAtStr) + sm.episodes = append(sm.episodes, models.EpisodeRecord{ + ID: id, + AgentID: agentID, + Namespace: namespace, + Content: content, + Category: category, + CreatedAt: createdAt, + }) + } + + return nil +} + +func float32sToFloat64s(f32 []float32) []float64 { + f64 := make([]float64, len(f32)) + for i := range f32 { + f64[i] = float64(f32[i]) + } + return f64 +} + +func (sm *SQLiteMemClient) persistMemory(e *memEntry) error { + vecJSON, _ := json.Marshal(float32sToFloat64s(e.Vector)) + vhJSON, _ := json.Marshal(e.VersionHistory) + createdAt := e.CreatedAt.Format(time.RFC3339) + _, err := sm.db.Exec(`INSERT OR REPLACE INTO memories + (id,content,agent_id,namespace,category,tier,quality_score,vector, + is_deleted,version,version_history,source,created_at,last_recalled_at, + recall_count,useful_count,not_useful_count) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + e.ID, e.Content, e.AgentID, e.Namespace, e.Category, e.Tier, + e.QualityScore, vecJSON, e.IsDeleted, e.Version, vhJSON, e.Source, + createdAt, e.LastRecalledAt, e.RecallCount, e.UsefulCount, e.NotUsefulCount) + return err +} + +func (sm *SQLiteMemClient) persistEpisode(r models.EpisodeRecord) error { + _, err := sm.db.Exec(`INSERT OR REPLACE INTO episodes (id,agent_id,namespace,content,category,created_at) + VALUES (?,?,?,?,?,?)`, + r.ID, r.AgentID, r.Namespace, r.Content, r.Category, r.CreatedAt.Format(time.RFC3339)) + return err +} + +// ─── Override MemLanceClient 方法，追加 SQLite 持久化 ─── + +func (sm *SQLiteMemClient) InsertMemory(m models.MemoryRecord) error { + // 先走内存逻辑（计算向量、写入内存 map） + if err := sm.MemLanceClient.InsertMemory(m); err != nil { + return err + } + // 同步写 SQLite + sm.mu.RLock() + e := sm.memories[m.ID] + sm.mu.RUnlock() + if e != nil { + return sm.persistMemory(e) + } + return nil +} + +func (sm *SQLiteMemClient) InsertEpisode(agentID, namespace, content, category string) (string, error) { + id, err := sm.MemLanceClient.InsertEpisode(agentID, namespace, content, category) + if err != nil { + return id, err + } + sm.mu.RLock() + for i := len(sm.episodes) - 1; i >= 0; i-- { + if sm.episodes[i].ID == id { + sm.persistEpisode(sm.episodes[i]) + break + } + } + sm.mu.RUnlock() + return id, nil +} + +func (sm *SQLiteMemClient) SoftDelete(id, reason string) error { + if err := sm.MemLanceClient.SoftDelete(id, reason); err != nil { + return err + } + sm.mu.RLock() + e := sm.memories[id] + sm.mu.RUnlock() + if e != nil { + return sm.persistMemory(e) + } + return nil +} + +func (sm *SQLiteMemClient) UpdateMemoryContent(id, newContent, source string) error { + if err := sm.MemLanceClient.UpdateMemoryContent(id, newContent, source); err != nil { + return err + } + sm.mu.RLock() + e := sm.memories[id] + sm.mu.RUnlock() + if e != nil { + return sm.persistMemory(e) + } + return nil +} + +func (sm *SQLiteMemClient) IncrementUseful(id string) { + sm.MemLanceClient.IncrementUseful(id) + sm.mu.RLock() + e := sm.memories[id] + sm.mu.RUnlock() + if e != nil { + sm.persistMemory(e) + } +} + +func (sm *SQLiteMemClient) IncrementNotUseful(id string) { + sm.MemLanceClient.IncrementNotUseful(id) + sm.mu.RLock() + e := sm.memories[id] + sm.mu.RUnlock() + if e != nil { + sm.persistMemory(e) + } +} + +func (sm *SQLiteMemClient) Update(table, id string, fields map[string]any) error { + if err := sm.MemLanceClient.Update(table, id, fields); err != nil { + return err + } + sm.mu.RLock() + e := sm.memories[id] + sm.mu.RUnlock() + if e != nil { + return sm.persistMemory(e) + } + return nil +} + +func (sm *SQLiteMemClient) Stats() (map[string]interface{}, error) { + sm.mu.RLock() + defer sm.mu.RUnlock() + return map[string]interface{}{ + "memory_count": len(sm.memories), + "episode_count": len(sm.episodes), + "backend": "sqlite+memvector (pure Go)", + }, nil +} \ No newline at end of file diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 0000000..601009e --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# ====== 织忆备份脚本 ====== +# 备份到 server Gitea（http://192.168.123.11:3000） +# 用法：./backup.sh [commit_message] +# 自动：每天 03:00（见 cron 配置） + +set -e + +DATE=$(date +%Y%m%d_%H%M%S) +LOG_FILE="${HOME}/.hermes/logs/backup.log" + +# 项目根目录（backup.sh 同级） +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# 备份临时目录 +BACKUP_DIR="/tmp/zhiyi_backup_${DATE}" +CONFIG_DIR="/tmp/zhiyi_config_${DATE}" + +# 日志 +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" +} + +log "===== 备份开始: ${DATE} =====" + +# 1. 备份数据文件 +log "[1/6] 备份数据文件..." +mkdir -p "${BACKUP_DIR}" +sudo rsync -a /var/lib/memoryweave/ "${BACKUP_DIR}/memoryweave/" --quiet +DATA_SIZE=$(du -sh "${BACKUP_DIR}" | cut -f1) +log " 数据备份完成: ${DATA_SIZE}" + +# 2. 打包 +log "[2/6] 打包数据..." +cd /tmp +tar -czf "zhiyi_backup_${DATE}.tar.gz" "zhiyi_backup_${DATE}/" +rm -rf "zhiyi_backup_${DATE}" +ARCHIVE_SIZE=$(du -sh "/tmp/zhiyi_backup_${DATE}.tar.gz" | cut -f1) +log " 打包完成: ${ARCHIVE_SIZE}" + +# 3. 备份 Hermes 配置 +log "[3/6] 备份 Hermes 配置..." +mkdir -p "${CONFIG_DIR}" + +# 核心人格文件 +rsync -a "${HOME}/.hermes/SOUL.md" "${CONFIG_DIR}/" 2>/dev/null || true +rsync -a "${HOME}/.hermes/AGENTS.md" "${CONFIG_DIR}/" 2>/dev/null || true +rsync -a "${HOME}/.hermes/MEMORY.md" "${CONFIG_DIR}/" 2>/dev/null || true +rsync -a "${HOME}/.hermes/USER.md" "${CONFIG_DIR}/" 2>/dev/null || true + +# 配置文件 +rsync -a "${HOME}/.hermes/config.yaml" "${CONFIG_DIR}/" 2>/dev/null || true +rsync -a "${HOME}/.hermes/.env" "${CONFIG_DIR}/" 2>/dev/null || true +rsync -a "${HOME}/.hermes/auth.json" "${CONFIG_DIR}/" 2>/dev/null || true + +# skills / cron / kanban +[ -d "${HOME}/.hermes/skills" ] && rsync -a "${HOME}/.hermes/skills/" "${CONFIG_DIR}/skills/" --quiet +[ -d "${HOME}/.hermes/cron" ] && rsync -a "${HOME}/.hermes/cron/" "${CONFIG_DIR}/cron/" --quiet +[ -f "${HOME}/.hermes/kanban.db" ] && rsync -a "${HOME}/.hermes/kanban.db" "${CONFIG_DIR}/" 2>/dev/null || true + +# memories / sessions（可选，较大） +if [ -d "${HOME}/.hermes/memories" ] && [ "$(du -sm "${HOME}/.hermes/memories" 2>/dev/null | cut -f1)" -lt 500 ]; then + rsync -a "${HOME}/.hermes/memories/" "${CONFIG_DIR}/memories/" --quiet +fi +if [ -d "${HOME}/.hermes/sessions" ] && [ "$(du -sm "${HOME}/.hermes/sessions" 2>/dev/null | cut -f1)" -lt 200 ]; then + rsync -a "${HOME}/.hermes/sessions/" "${CONFIG_DIR}/sessions/" --quiet +fi + +CONFIG_SIZE=$(du -sh "${CONFIG_DIR}" | cut -f1) +log " 配置备份完成: ${CONFIG_SIZE}" + +# 4. 推送到 Gitea +log "[4/6] 推送到 Gitea..." + +# 确保 git 仓库存在 +if [ ! -d "${PROJECT_ROOT}/.git" ]; then + log " WARM: ${PROJECT_ROOT} 不是 git 仓库，初始化..." + cd "${PROJECT_ROOT}" + git init + git remote add origin "http://192.168.123.11:3000/xiaoxue_admin/zhiyi-config.git" 2>/dev/null || true +fi + +cd "${PROJECT_ROOT}" + +# 确保 data 目录存在 +mkdir -p data + +# 移动备份到 data 目录 +mv "/tmp/zhiyi_backup_${DATE}.tar.gz" data/ + +# 把数据仓库也 clone/update +if [ ! -d "${PROJECT_ROOT}/zhiyi-backup" ]; then + git clone "http://192.168.123.11:3000/xiaoxue_admin/zhiyi-backup.git" "${PROJECT_ROOT}/zhiyi-backup" 2>/dev/null || { + # 如果仓库不存在，跳过数据备份 + log " WARN: zhiyi-backup 仓库不存在或无法访问，跳过数据推送" + } +fi + +if [ -d "${PROJECT_ROOT}/zhiyi-backup" ]; then + cd "${PROJECT_ROOT}/zhiyi-backup" + # 移动 archive + if [ -f "${PROJECT_ROOT}/data/zhiyi_backup_${DATE}.tar.gz" ]; then + cp "${PROJECT_ROOT}/data/zhiyi_backup_${DATE}.tar.gz" . + git add "zhiyi_backup_${DATE}.tar.gz" + git commit -m "Backup data ${DATE}" 2>/dev/null || true + git push origin main 2>/dev/null || log " WARN: 数据推送失败（可能需要 token）" + rm -f "zhiyi_backup_${DATE}.tar.gz" + fi + cd "${PROJECT_ROOT}" +fi + +# 推送配置 +git add data/zhiyi_backup_${DATE}.tar.gz +git add config/ 2>/dev/null || true +git add -A +COMMIT_MSG="${1:-Backup ${DATE}}" +git commit -m "${COMMIT_MSG}" 2>/dev/null || log " WARN: 没有新内容需要提交" + +git push origin main 2>/dev/null || log " WARN: 配置推送失败（可能需要 token）" + +# 5. 清理 +log "[5/6] 清理临时文件..." +rm -rf "${BACKUP_DIR}" "${CONFIG_DIR}" +log " 清理完成" + +# 6. 完成 +log "[6/6] 备份完成" +log "===== 备份结束: ${DATE} =====" +echo "" \ No newline at end of file diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 0000000..384ab5f --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# ====== 织忆恢复脚本 ====== +# 从 server Gitea 恢复所有数据 +# 用法：./restore.sh [backup_date] +# backup_date: 可选，默认最新 + +set -e + +BACKUP_DATE="${1:-latest}" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GITEA_BASE="http://192.168.123.11:3000/xiaoxue_admin" + +echo "[$(date '+%Y-%m-%d %H:%M:%S')] ===== 开始恢复织忆 =====" + +# 1. Clone 配置仓库 +echo "[1/5] 拉取 Hermes 配置..." +if [ -d "${PROJECT_ROOT}/zhiyi-config" ]; then + cd "${PROJECT_ROOT}/zhiyi-config" + git pull origin main +else + git clone "${GITEA_BASE}/zhiyi-config.git" "${PROJECT_ROOT}/zhiyi-config" +fi + +# 2. 恢复核心人格文件 +echo "[2/5] 恢复人格文件..." +cp "${PROJECT_ROOT}/zhiyi-config/SOUL.md" "${HOME}/.hermes/SOUL.md" +cp "${PROJECT_ROOT}/zhiyi-config/AGENTS.md" "${HOME}/.hermes/AGENTS.md" +cp "${PROJECT_ROOT}/zhiyi-config/MEMORY.md" "${HOME}/.hermes/MEMORY.md" +cp "${PROJECT_ROOT}/zhiyi-config/USER.md" "${HOME}/.hermes/USER.md" +echo " 人格文件恢复完成" + +# 3. 恢复配置文件 +echo "[3/5] 恢复配置文件..." +[ -f "${PROJECT_ROOT}/zhiyi-config/config.yaml" ] && \ + cp "${PROJECT_ROOT}/zhiyi-config/config.yaml" "${HOME}/.hermes/config.yaml" +[ -f "${PROJECT_ROOT}/zhiyi-config/.env" ] && \ + cp "${PROJECT_ROOT}/zhiyi-config/.env" "${HOME}/.hermes/.env" +[ -f "${PROJECT_ROOT}/zhiyi-config/auth.json" ] && \ + cp "${PROJECT_ROOT}/zhiyi-config/auth.json" "${HOME}/.hermes/auth.json" +echo " 配置文件恢复完成" + +# 4. 恢复 skills / cron +echo "[4/5] 恢复 skills 和 cron..." +[ -d "${PROJECT_ROOT}/zhiyi-config/skills" ] && \ + rsync -a "${PROJECT_ROOT}/zhiyi-config/skills/" "${HOME}/.hermes/skills/" +[ -d "${PROJECT_ROOT}/zhiyi-config/cron" ] && \ + rsync -a "${PROJECT_ROOT}/zhiyi-config/cron/" "${HOME}/.hermes/cron/" +[ -f "${PROJECT_ROOT}/zhiyi-config/kanban.db" ] && \ + cp "${PROJECT_ROOT}/zhiyi-config/kanban.db" "${HOME}/.hermes/kanban.db" +echo " skills/cron 恢复完成" + +# 5. 恢复织忆数据 +echo "[5/5] 恢复织忆数据..." + +# Clone 数据仓库（如果需要） +if [ ! -d "${PROJECT_ROOT}/zhiyi-backup" ]; then + git clone "${GITEA_BASE}/zhiyi-backup.git" "${PROJECT_ROOT}/zhiyi-backup" +fi + +cd "${PROJECT_ROOT}/zhiyi-backup" + +# 找最新备份 +if [ "${BACKUP_DATE}" = "latest" ]; then + BACKUP_FILE=$(ls -t *.tar.gz 2>/dev/null | head -1) +else + BACKUP_FILE="zhiyi_backup_${BACKUP_DATE}.tar.gz" +fi + +if [ -n "${BACKUP_FILE}" ] && [ -f "${BACKUP_FILE}" ]; then + echo " 使用备份: ${BACKUP_FILE}" + + # 解压到临时目录 + TMP_DIR="/tmp/zhiyi_restore_$$" + mkdir -p "${TMP_DIR}" + tar -xzf "${BACKUP_FILE}" -C "${TMP_DIR}" + + DATA_DIR=$(ls -d "${TMP_DIR}"/zhiyi_backup_* 2>/dev/null | head -1) + + if [ -n "${DATA_DIR}" ] && [ -d "${DATA_DIR}/memoryweave" ]; then + sudo rsync -a "${DATA_DIR}/memoryweave/" /var/lib/memoryweave/ + sudo chown -R root:root /var/lib/memoryweave/ + echo " 数据恢复完成: $(du -sh /var/lib/memoryweave/ | cut -f1)" + else + echo " ERROR: 备份格式错误，无法恢复" + fi + + rm -rf "${TMP_DIR}" +else + echo " WARN: 未找到备份文件 ${BACKUP_FILE}，跳过数据恢复" +fi + +# 6. 重启服务 +echo "" +echo "[完成] 准备重启服务..." + +# 检查 zhiyi 进程 +if pgrep -f "zhiyi" > /dev/null; then + echo " 重启 zhiyi..." + pkill -f "zhiyi" 2>/dev/null || true + sleep 2 +fi + +# 启动 zhiyi +cd "${PROJECT_ROOT}/go" +if [ -f "./zhiyi" ]; then + nohup ./zhiyi >> "${HOME}/.hermes/logs/zhiyi.log" 2>&1 & + sleep 3 +else + echo " WARN: zhiyi binary 不存在，需要重新编译" +fi + +# 7. 健康检查 +echo "" +echo "健康检查..." +sleep 2 +if curl -s http://localhost:7821/health > /dev/null 2>&1; then + echo "✓ 织忆服务正常" +else + echo "✗ 织忆服务异常，请检查日志" + tail -20 "${HOME}/.hermes/logs/zhiyi.log" 2>/dev/null || true +fi + +echo "" +echo "[$(date '+%Y-%m-%d %H:%M:%S')] ===== 恢复完成 =====" \ No newline at end of file diff --git a/scripts/verify_backup.sh b/scripts/verify_backup.sh new file mode 100755 index 0000000..a8eb6ea --- /dev/null +++ b/scripts/verify_backup.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# ====== 备份验证脚本 ====== +# 每周日凌晨 04:00 自动运行 +# 验证备份完整性，解压测试 SQLite 数据库 + +set -e + +DATE=$(date +%Y%m%d) +VERIFY_DIR="/tmp/zhiyi_verify_${DATE}" +LOG_FILE="${HOME}/.hermes/logs/backup_verify.log" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" +} + +log "===== 备份验证开始 =====" + +# 1. Clone backup 仓库 +log "[1/4] 拉取备份..." +if [ ! -d "${HOME}/zhiyi-backup-verify" ]; then + git clone "http://192.168.123.11:3000/xiaoxue_admin/zhiyi-backup.git" "${HOME}/zhiyi-backup-verify" +else + cd "${HOME}/zhiyi-backup-verify" + git pull origin main +fi + +cd "${HOME}/zhiyi-backup-verify" + +# 找最新备份 +LATEST=$(ls -t *.tar.gz 2>/dev/null | head -1) +if [ -z "$LATEST" ]; then + log "ERROR: 未找到任何备份文件" + exit 1 +fi +log "验证备份: ${LATEST}" + +# 2. 校验 tar.gz 完整性 +log "[2/4] 校验 tar.gz 完整性..." +if tar -tzf "$LATEST" > /dev/null 2>&1; then + log "✓ tar.gz 文件完整" +else + log "ERROR: tar.gz 文件损坏" + exit 1 +fi + +# 3. 解压并校验 SQLite +log "[3/4] 解压并校验 SQLite..." +mkdir -p "${VERIFY_DIR}" +tar -xzf "$LATEST" -C "${VERIFY_DIR}" + +DATA_DIR=$(ls -d "${VERIFY_DIR}"/zhiyi_backup_* 2>/dev/null | head -1) +if [ -z "$DATA_DIR" ]; then + log "ERROR: 无法找到解压后的数据目录" + exit 1 +fi + +# 校验 graph.db +if [ -f "${DATA_DIR}/memoryweave/graph.db" ]; then + NODE_COUNT=$(sqlite3 "${DATA_DIR}/memoryweave/graph.db" "SELECT count(*) FROM nodes;" 2>/dev/null || echo "ERR") + if [ "$NODE_COUNT" != "ERR" ]; then + log "✓ 图谱数据库正常 (${NODE_COUNT} nodes)" + else + log "ERROR: 图谱数据库损坏" + fi +else + log "WARN: graph.db 不存在" +fi + +# 校验 memoryweave.db +if [ -f "${DATA_DIR}/memoryweave/memoryweave.db" ]; then + DB_SIZE=$(du -sh "${DATA_DIR}/memoryweave/memoryweave.db" | cut -f1) + log "✓ LiteDB 数据库存在 (${DB_SIZE})" +else + log "WARN: memoryweave.db 不存在" +fi + +# 校验 LanceDB 目录 +if [ -d "${DATA_DIR}/memoryweave/memories.lance" ]; then + LANCE_SIZE=$(du -sh "${DATA_DIR}/memoryweave/memories.lance" | cut -f1) + log "✓ LanceDB 向量目录存在 (${LANCE_SIZE})" +else + log "WARN: memories.lance 不存在" +fi + +# 4. 清理 +log "[4/4] 清理临时文件..." +rm -rf "${VERIFY_DIR}" +log "===== 备份验证完成 =====" +echo "" \ No newline at end of file