病根:两个列表/导出端点一直用『零向量 ANN』冒充『列出全部』——零向量没有方向, ANN 退化成本存储序前 N 行,再按 ns 客户端过滤 ⇒ 计数完全失真 (实测 export hermes-main 报 1 条,真实 139+;shared 报 0,真实 29)。 修复: ① Rust lancedb_ops: scan_for_forgetting 加 namespace 下推(only_if),limit 作用于过滤后结果集 ② Go 接口 LanceDB 新增 Scan(limit, ns): - RustLanceDBClient → lancedb_scan RPC - SQLiteClient → SELECT ... WHERE is_deleted=0 [AND ns] ORDER BY created_at DESC LIMIT - MemLanceClient → 内存 map 过滤 + created_at 倒序 ③ 两个端点 ListMemories / ExportMemoriesMD 改用 aa.LanceDB.Scan(limit, ns) ④ 补 mock(zz_commit mockLance / write-buffer fakeLDB)+ 5 条新判据 (全表 3 / hermes-main 2 / shared 1 / limit 生效 / 不存在 ns 0)
810 lines
23 KiB
Go
810 lines
23 KiB
Go
// 织忆 MemoryWeave — SQLite 持久存储后端(CGO,实现 LanceDB 接口)
|
|
package storage
|
|
|
|
/*
|
|
#cgo LDFLAGS: -lsqlite3
|
|
#include <sqlite3.h>
|
|
#include <stdlib.h>
|
|
*/
|
|
import "C"
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
"fmt"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
"unsafe"
|
|
|
|
"github.com/xiaoxue/memoryweave/internal/models"
|
|
)
|
|
|
|
// SQLiteClient CGO 直连 SQLite3,实现 LanceDB 接口。
|
|
// 生产环境持久化存储,替代 MemLanceClient。
|
|
type SQLiteClient struct {
|
|
mu sync.RWMutex
|
|
db *C.sqlite3
|
|
path string
|
|
}
|
|
|
|
func NewSQLiteClient(dbPath string) (*SQLiteClient, error) {
|
|
if dbPath == "" {
|
|
dbPath = "/var/lib/zhiyi/data/memoryweave.db"
|
|
}
|
|
|
|
cPath := C.CString(dbPath)
|
|
defer C.free(unsafe.Pointer(cPath))
|
|
|
|
var db *C.sqlite3
|
|
rc := C.sqlite3_open(cPath, &db)
|
|
if rc != C.SQLITE_OK {
|
|
errMsg := C.GoString(C.sqlite3_errmsg(db))
|
|
C.sqlite3_close(db)
|
|
return nil, fmt.Errorf("sqlite open: %s", errMsg)
|
|
}
|
|
|
|
// 性能优化
|
|
C.sqlite3_exec(db, C.CString("PRAGMA journal_mode=WAL"), nil, nil, nil)
|
|
C.sqlite3_exec(db, C.CString("PRAGMA synchronous=NORMAL"), nil, nil, nil)
|
|
C.sqlite3_exec(db, C.CString("PRAGMA busy_timeout=5000"), nil, nil, nil)
|
|
C.sqlite3_exec(db, C.CString("PRAGMA cache_size=-20000"), nil, nil, nil)
|
|
|
|
sc := &SQLiteClient{db: db, path: dbPath}
|
|
if err := sc.migrate(); err != nil {
|
|
C.sqlite3_close(db)
|
|
return nil, err
|
|
}
|
|
return sc, nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) Close() error {
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
if sc.db != nil {
|
|
C.sqlite3_close(sc.db)
|
|
sc.db = nil
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ─── migration ────────────────────────────────────────
|
|
|
|
func (sc *SQLiteClient) migrate() error {
|
|
sqls := []string{
|
|
`CREATE TABLE IF NOT EXISTS memories (
|
|
id TEXT PRIMARY KEY,
|
|
agent_id TEXT NOT NULL,
|
|
namespace TEXT NOT NULL DEFAULT '',
|
|
content TEXT NOT NULL,
|
|
category TEXT NOT NULL DEFAULT 'general',
|
|
vector BLOB,
|
|
tier TEXT NOT NULL DEFAULT 'normal',
|
|
importance REAL DEFAULT 1.0,
|
|
quality_score REAL DEFAULT 0.0,
|
|
recall_count INTEGER DEFAULT 0,
|
|
useful_count INTEGER DEFAULT 0,
|
|
not_useful_count INTEGER DEFAULT 0,
|
|
freshness TEXT DEFAULT 'fresh',
|
|
version INTEGER DEFAULT 1,
|
|
version_history TEXT DEFAULT '[]',
|
|
source TEXT DEFAULT '',
|
|
volatile_flag INTEGER DEFAULT 0,
|
|
is_deleted INTEGER DEFAULT 0,
|
|
depends_on TEXT DEFAULT '[]',
|
|
derived_from TEXT DEFAULT '',
|
|
last_recalled_at TEXT DEFAULT '',
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
)`,
|
|
`CREATE TABLE IF NOT EXISTS episodes (
|
|
id TEXT PRIMARY KEY,
|
|
agent_id TEXT NOT NULL,
|
|
namespace TEXT NOT NULL DEFAULT '',
|
|
content TEXT NOT NULL,
|
|
category TEXT NOT NULL DEFAULT 'general',
|
|
distilled_to TEXT DEFAULT '',
|
|
distill_status TEXT DEFAULT 'pending',
|
|
created_at TEXT NOT NULL
|
|
)`,
|
|
`CREATE TABLE IF NOT EXISTS tombstones (
|
|
id TEXT PRIMARY KEY,
|
|
original_id TEXT NOT NULL,
|
|
content_snapshot TEXT DEFAULT '',
|
|
namespace TEXT DEFAULT '',
|
|
reason TEXT NOT NULL DEFAULT '',
|
|
merged_into TEXT DEFAULT '',
|
|
deleted_at TEXT NOT NULL
|
|
)`,
|
|
`CREATE TABLE IF NOT EXISTS version_history (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
memory_id TEXT NOT NULL,
|
|
version INTEGER NOT NULL,
|
|
content TEXT NOT NULL,
|
|
source TEXT DEFAULT '',
|
|
reason TEXT DEFAULT '',
|
|
timestamp TEXT NOT NULL,
|
|
FOREIGN KEY (memory_id) REFERENCES memories(id)
|
|
)`,
|
|
`CREATE INDEX IF NOT EXISTS idx_memories_agent ON memories(agent_id)`,
|
|
`CREATE INDEX IF NOT EXISTS idx_memories_namespace ON memories(namespace)`,
|
|
`CREATE INDEX IF NOT EXISTS idx_memories_deleted ON memories(is_deleted)`,
|
|
`CREATE INDEX IF NOT EXISTS idx_episodes_agent ON episodes(agent_id)`,
|
|
`CREATE INDEX IF NOT EXISTS idx_version_history_mem ON version_history(memory_id)`,
|
|
}
|
|
|
|
for _, s := range sqls {
|
|
if err := sc.exec(s); err != nil {
|
|
return fmt.Errorf("migrate: %w", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ─── CGO helpers ──────────────────────────────────────
|
|
|
|
func (sc *SQLiteClient) exec(sql string) error {
|
|
cSQL := C.CString(sql)
|
|
defer C.free(unsafe.Pointer(cSQL))
|
|
|
|
var errMsg *C.char
|
|
rc := C.sqlite3_exec(sc.db, cSQL, nil, nil, &errMsg)
|
|
if rc != C.SQLITE_OK {
|
|
msg := C.GoString(errMsg)
|
|
C.sqlite3_free(unsafe.Pointer(errMsg))
|
|
return fmt.Errorf("sqlite: %s", msg)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) execWithArgs(sql string, args ...interface{}) error {
|
|
cSQL := C.CString(sql)
|
|
defer C.free(unsafe.Pointer(cSQL))
|
|
|
|
var stmt *C.sqlite3_stmt
|
|
rc := C.sqlite3_prepare_v2(sc.db, cSQL, C.int(len(sql)), &stmt, nil)
|
|
if rc != C.SQLITE_OK {
|
|
return fmt.Errorf("sqlite prepare: %s", C.GoString(C.sqlite3_errmsg(sc.db)))
|
|
}
|
|
defer C.sqlite3_finalize(stmt)
|
|
|
|
for i, arg := range args {
|
|
idx := C.int(i + 1)
|
|
switch v := arg.(type) {
|
|
case nil:
|
|
C.sqlite3_bind_null(stmt, idx)
|
|
case int:
|
|
C.sqlite3_bind_int64(stmt, idx, C.sqlite3_int64(v))
|
|
case int64:
|
|
C.sqlite3_bind_int64(stmt, idx, C.sqlite3_int64(v))
|
|
case float64:
|
|
C.sqlite3_bind_double(stmt, idx, C.double(v))
|
|
case bool:
|
|
if v {
|
|
C.sqlite3_bind_int(stmt, idx, 1)
|
|
} else {
|
|
C.sqlite3_bind_int(stmt, idx, 0)
|
|
}
|
|
case string:
|
|
cStr := C.CString(v)
|
|
C.sqlite3_bind_text(stmt, idx, cStr, C.int(len(v)), (*[0]byte)(C.free))
|
|
case []byte:
|
|
if len(v) == 0 {
|
|
C.sqlite3_bind_null(stmt, idx)
|
|
} else {
|
|
C.sqlite3_bind_blob(stmt, idx, unsafe.Pointer(&v[0]), C.int(len(v)), nil)
|
|
}
|
|
default:
|
|
return fmt.Errorf("unsupported arg type %T", arg)
|
|
}
|
|
}
|
|
|
|
rc = C.sqlite3_step(stmt)
|
|
if rc != C.SQLITE_DONE && rc != C.SQLITE_ROW {
|
|
return fmt.Errorf("sqlite step: %s", C.GoString(C.sqlite3_errmsg(sc.db)))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type sqliteRow map[string]interface{}
|
|
|
|
func (sc *SQLiteClient) query(sql string) ([]sqliteRow, error) {
|
|
cSQL := C.CString(sql)
|
|
defer C.free(unsafe.Pointer(cSQL))
|
|
|
|
var stmt *C.sqlite3_stmt
|
|
rc := C.sqlite3_prepare_v2(sc.db, cSQL, C.int(len(sql)), &stmt, nil)
|
|
if rc != C.SQLITE_OK {
|
|
return nil, fmt.Errorf("sqlite prepare: %s", C.GoString(C.sqlite3_errmsg(sc.db)))
|
|
}
|
|
defer C.sqlite3_finalize(stmt)
|
|
|
|
var rows []sqliteRow
|
|
for {
|
|
rc = C.sqlite3_step(stmt)
|
|
if rc == C.SQLITE_DONE {
|
|
break
|
|
}
|
|
if rc != C.SQLITE_ROW {
|
|
return nil, fmt.Errorf("sqlite step: %s", C.GoString(C.sqlite3_errmsg(sc.db)))
|
|
}
|
|
|
|
colCount := int(C.sqlite3_column_count(stmt))
|
|
row := make(sqliteRow, colCount)
|
|
for i := 0; i < colCount; i++ {
|
|
name := C.GoString(C.sqlite3_column_name(stmt, C.int(i)))
|
|
colType := C.sqlite3_column_type(stmt, C.int(i))
|
|
switch colType {
|
|
case C.SQLITE_INTEGER:
|
|
row[name] = int64(C.sqlite3_column_int64(stmt, C.int(i)))
|
|
case C.SQLITE_FLOAT:
|
|
row[name] = float64(C.sqlite3_column_double(stmt, C.int(i)))
|
|
case C.SQLITE_TEXT:
|
|
row[name] = C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, C.int(i)))))
|
|
case C.SQLITE_BLOB:
|
|
n := int(C.sqlite3_column_bytes(stmt, C.int(i)))
|
|
ptr := C.sqlite3_column_blob(stmt, C.int(i))
|
|
blob := make([]byte, n)
|
|
if n > 0 {
|
|
copy(blob, (*[1 << 30]byte)(ptr)[:n])
|
|
}
|
|
row[name] = blob
|
|
case C.SQLITE_NULL:
|
|
row[name] = nil
|
|
}
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
return rows, nil
|
|
}
|
|
|
|
// ─── vector helpers ───────────────────────────────────
|
|
|
|
func vectorToBlob(v []float32) []byte {
|
|
buf := new(bytes.Buffer)
|
|
for _, f := range v {
|
|
binary.Write(buf, binary.LittleEndian, f)
|
|
}
|
|
return buf.Bytes()
|
|
}
|
|
|
|
func blobToVector(b []byte) []float32 {
|
|
if len(b) == 0 {
|
|
return nil
|
|
}
|
|
vec := make([]float32, len(b)/4)
|
|
buf := bytes.NewReader(b)
|
|
for i := range vec {
|
|
binary.Read(buf, binary.LittleEndian, &vec[i])
|
|
}
|
|
return vec
|
|
}
|
|
|
|
// ─── LanceDB 接口 ─────────────────────────────────────
|
|
|
|
func (sc *SQLiteClient) InsertEpisode(agentID, namespace, content, category string) (string, error) {
|
|
id := fmt.Sprintf("ep_%d", time.Now().UnixNano())
|
|
now := time.Now().Format(time.RFC3339)
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
return id, sc.execWithArgs(
|
|
"INSERT INTO episodes(id,agent_id,namespace,content,category,created_at) VALUES(?,?,?,?,?,?)",
|
|
id, agentID, namespace, content, category, now,
|
|
)
|
|
}
|
|
|
|
func (sc *SQLiteClient) InsertMemory(m models.MemoryRecord) error {
|
|
now := time.Now().Format(time.RFC3339)
|
|
vecBlob := vectorToBlob(m.Vector)
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
return sc.execWithArgs(
|
|
`INSERT INTO memories(id,agent_id,namespace,content,category,vector,tier,importance,quality_score,
|
|
source,depends_on,derived_from,version,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,1,?,?)`,
|
|
m.ID, m.AgentID, m.Namespace, m.Content, m.Category, vecBlob, m.Tier,
|
|
1.0, m.QualityScore, "", "[]", "", now, now,
|
|
)
|
|
}
|
|
|
|
func (sc *SQLiteClient) Insert(table string, record any) error {
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
|
|
now := time.Now().Format(time.RFC3339)
|
|
switch r := record.(type) {
|
|
case models.MemoryRecord:
|
|
return sc.InsertMemory(r)
|
|
case models.EpisodeRecord:
|
|
return sc.execWithArgs(
|
|
"INSERT INTO episodes(id,agent_id,namespace,content,category,distilled_to,distill_status,created_at) VALUES(?,?,?,?,?,?,?,?)",
|
|
r.ID, r.AgentID, r.Namespace, r.Content, r.Category, "", "pending", now,
|
|
)
|
|
case models.TombstoneRecord:
|
|
delTime := r.CreatedAt.Format(time.RFC3339)
|
|
return sc.execWithArgs(
|
|
"INSERT INTO tombstones(id,original_id,content_snapshot,namespace,reason,merged_into,deleted_at) VALUES(?,?,?,?,?,?,?)",
|
|
r.ID, r.OriginalID, "", "", r.Reason, "", delTime,
|
|
)
|
|
default:
|
|
return fmt.Errorf("sqlite: unsupported record type %T", record)
|
|
}
|
|
}
|
|
|
|
// Scan SQLite 后端的全表扫描(is_deleted=0 + 可选 namespace,按 created_at 倒序)。
|
|
func (sc *SQLiteClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
|
if limit <= 0 {
|
|
limit = 100
|
|
}
|
|
sc.mu.RLock()
|
|
defer sc.mu.RUnlock()
|
|
|
|
q := "SELECT id, content, category, namespace, importance, quality_score, recall_count, tier, source, created_at, updated_at FROM memories WHERE is_deleted=0"
|
|
if namespaceFilter != "" {
|
|
q += " AND namespace='" + sanitizeIdent(namespaceFilter) + "'"
|
|
}
|
|
q += " ORDER BY created_at DESC LIMIT " + fmt.Sprintf("%d", limit)
|
|
|
|
rows, err := sc.query(q)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]models.MemoryRecord, 0, len(rows))
|
|
for _, row := range rows {
|
|
rec := models.MemoryRecord{}
|
|
rec.ID, _ = row["id"].(string)
|
|
rec.Content, _ = row["content"].(string)
|
|
rec.Category, _ = row["category"].(string)
|
|
rec.Namespace, _ = row["namespace"].(string)
|
|
rec.Importance, _ = row["importance"].(float64)
|
|
rec.QualityScore, _ = row["quality_score"].(float64)
|
|
rec.Tier, _ = row["tier"].(string)
|
|
rec.Source, _ = row["source"].(string)
|
|
if n, ok := row["recall_count"].(int64); ok {
|
|
rec.RecallCount = int(n)
|
|
}
|
|
if ts, ok := row["created_at"].(string); ok {
|
|
if t, e := time.Parse(time.RFC3339, ts); e == nil {
|
|
rec.CreatedAt = t
|
|
}
|
|
}
|
|
if ts, ok := row["updated_at"].(string); ok {
|
|
if t, e := time.Parse(time.RFC3339, ts); e == nil {
|
|
rec.UpdatedAt = t
|
|
}
|
|
}
|
|
out = append(out, rec)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
|
sc.mu.RLock()
|
|
defer sc.mu.RUnlock()
|
|
|
|
// 加载所有未被删除的记录 + 向量 (按 importance 加权排序)
|
|
query := "SELECT id, content, category, namespace, vector, importance, created_at FROM memories WHERE is_deleted=0"
|
|
if namespaceFilter != "" {
|
|
query += " AND namespace='" + sanitizeIdent(namespaceFilter) + "'"
|
|
}
|
|
|
|
rows, err := sc.query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
type candidate struct {
|
|
rec models.MemoryRecord
|
|
score float64
|
|
}
|
|
var candidates []candidate
|
|
for _, row := range rows {
|
|
vecBlob, _ := row["vector"].([]byte)
|
|
candVec := blobToVector(vecBlob)
|
|
sim := cosineSim(vector, candVec)
|
|
importance, _ := row["importance"].(float64)
|
|
if importance <= 0 {
|
|
importance = 1.0
|
|
}
|
|
|
|
createdAt, _ := time.Parse(time.RFC3339, strVal(row, "created_at"))
|
|
candidates = append(candidates, candidate{
|
|
rec: models.MemoryRecord{
|
|
ID: strVal(row, "id"),
|
|
Content: strVal(row, "content"),
|
|
Category: strVal(row, "category"),
|
|
Namespace: strVal(row, "namespace"),
|
|
CreatedAt: createdAt,
|
|
},
|
|
score: sim * importance, // importance 加权
|
|
})
|
|
}
|
|
|
|
sort.Slice(candidates, func(i, j int) bool {
|
|
return candidates[i].score > candidates[j].score
|
|
})
|
|
if topK > len(candidates) {
|
|
topK = len(candidates)
|
|
}
|
|
results := make([]models.MemoryRecord, topK)
|
|
for i := 0; i < topK; i++ {
|
|
results[i] = candidates[i].rec
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) GetTopByQuality(agentID string, limit int) ([]models.MemoryRecord, error) {
|
|
sc.mu.RLock()
|
|
defer sc.mu.RUnlock()
|
|
|
|
rows, err := sc.query(
|
|
"SELECT id,content,category,namespace,created_at FROM memories WHERE is_deleted=0 ORDER BY quality_score DESC LIMIT " + itoa(limit),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return sc.rowsToMemories(rows), nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) Stats() (map[string]interface{}, error) {
|
|
sc.mu.RLock()
|
|
defer sc.mu.RUnlock()
|
|
|
|
var memCount, epCount, tombCount int64
|
|
rows, _ := sc.query("SELECT COUNT(*) as cnt FROM memories WHERE is_deleted=0")
|
|
if len(rows) > 0 {
|
|
memCount, _ = rows[0]["cnt"].(int64)
|
|
}
|
|
rows, _ = sc.query("SELECT COUNT(*) as cnt FROM episodes")
|
|
if len(rows) > 0 {
|
|
epCount, _ = rows[0]["cnt"].(int64)
|
|
}
|
|
rows, _ = sc.query("SELECT COUNT(*) as cnt FROM tombstones")
|
|
if len(rows) > 0 {
|
|
tombCount, _ = rows[0]["cnt"].(int64)
|
|
}
|
|
|
|
return map[string]interface{}{
|
|
"memory_count": memCount,
|
|
"episode_count": epCount,
|
|
"tombstone_count": tombCount,
|
|
"backend": "sqlite3 (CGO)",
|
|
"db_path": sc.path,
|
|
}, nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) SoftDelete(id, reason string) error {
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
|
|
now := time.Now().Format(time.RFC3339)
|
|
if err := sc.execWithArgs(
|
|
"UPDATE memories SET is_deleted=1, updated_at=? WHERE id=?",
|
|
now, id,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
tombID := fmt.Sprintf("tomb_%d", time.Now().UnixNano())
|
|
return sc.execWithArgs(
|
|
"INSERT INTO tombstones(id,original_id,reason,deleted_at) VALUES(?,?,?,?)",
|
|
tombID, id, reason, now,
|
|
)
|
|
}
|
|
|
|
func (sc *SQLiteClient) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) {
|
|
if table != "memories" {
|
|
return 0, fmt.Errorf("unsupported table: %s", table)
|
|
}
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
var updated int64
|
|
for _, id := range ids {
|
|
res := sc.execWithArgs(
|
|
"UPDATE memories SET is_deleted=1, updated_at=? WHERE id=?",
|
|
updatedAt, id,
|
|
)
|
|
if res != nil {
|
|
return updated, res
|
|
}
|
|
updated++
|
|
}
|
|
return updated, nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) GetVersionHistory(id string) ([]map[string]interface{}, error) {
|
|
sc.mu.RLock()
|
|
defer sc.mu.RUnlock()
|
|
|
|
rows, err := sc.query(
|
|
"SELECT version,content,source,reason,timestamp FROM version_history WHERE memory_id=? ORDER BY version DESC",
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// 用笨办法实现参数化查询避免重写 query 函数
|
|
if len(rows) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
// 直接重新查询带参数
|
|
cSQL := C.CString("SELECT version,content,source,reason,timestamp FROM version_history WHERE memory_id=? ORDER BY version DESC")
|
|
defer C.free(unsafe.Pointer(cSQL))
|
|
var stmt *C.sqlite3_stmt
|
|
C.sqlite3_prepare_v2(sc.db, cSQL, -1, &stmt, nil)
|
|
defer C.sqlite3_finalize(stmt)
|
|
cID := C.CString(id)
|
|
defer C.free(unsafe.Pointer(cID))
|
|
C.sqlite3_bind_text(stmt, 1, cID, C.int(len(id)), (*[0]byte)(C.free))
|
|
|
|
var result []map[string]interface{}
|
|
for C.sqlite3_step(stmt) == C.SQLITE_ROW {
|
|
result = append(result, map[string]interface{}{
|
|
"version": int64(C.sqlite3_column_int64(stmt, 0)),
|
|
"content": C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 1)))),
|
|
"source": C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 2)))),
|
|
"reason": C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 3)))),
|
|
"timestamp": C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 4)))),
|
|
})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) GetCandidatesForForgetting() ([]map[string]interface{}, error) {
|
|
sc.mu.RLock()
|
|
defer sc.mu.RUnlock()
|
|
|
|
rows, err := sc.query(
|
|
"SELECT id, content, namespace, category, last_recalled_at, recall_count, tier, importance, created_at FROM memories WHERE is_deleted=0 AND tier!='core' ORDER BY last_recalled_at ASC LIMIT 200",
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var results []map[string]interface{}
|
|
for _, r := range rows {
|
|
results = append(results, map[string]interface{}{
|
|
"id": r["id"],
|
|
"content": r["content"],
|
|
"namespace": r["namespace"],
|
|
"category": r["category"],
|
|
"last_recalled_at": r["last_recalled_at"],
|
|
"recall_count": r["recall_count"],
|
|
"tier": r["tier"],
|
|
"importance": r["importance"],
|
|
"created_at": r["created_at"],
|
|
})
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
// G7.2: GetSkillCandidates 返回适合结晶为 skill 的记忆候选
|
|
// 条件: recall_count >= minRecalls, quality_score >= 0.7, tier != 'core'
|
|
func (sc *SQLiteClient) GetSkillCandidates(minRecalls, limit int) ([]models.MemoryRecord, error) {
|
|
sc.mu.RLock()
|
|
defer sc.mu.RUnlock()
|
|
|
|
sql := `SELECT id, agent_id, namespace, content, category,
|
|
recall_count, quality_score, importance, tier,
|
|
last_recalled_at, created_at, updated_at
|
|
FROM memories
|
|
WHERE is_deleted = 0 AND tier != 'core'
|
|
AND recall_count >= ? AND quality_score >= 0.7
|
|
ORDER BY recall_count DESC, quality_score DESC
|
|
LIMIT ?`
|
|
cSQL := C.CString(sql)
|
|
defer C.free(unsafe.Pointer(cSQL))
|
|
|
|
var stmt *C.sqlite3_stmt
|
|
rc := C.sqlite3_prepare_v2(sc.db, cSQL, C.int(len(sql)), &stmt, nil)
|
|
if rc != C.SQLITE_OK {
|
|
return nil, fmt.Errorf("prepare skill candidates: %s", C.GoString(C.sqlite3_errmsg(sc.db)))
|
|
}
|
|
defer C.sqlite3_finalize(stmt)
|
|
|
|
C.sqlite3_bind_int(stmt, 1, C.int(minRecalls))
|
|
C.sqlite3_bind_int(stmt, 2, C.int(limit))
|
|
|
|
var results []models.MemoryRecord
|
|
for {
|
|
rc = C.sqlite3_step(stmt)
|
|
if rc == C.SQLITE_DONE {
|
|
break
|
|
}
|
|
if rc != C.SQLITE_ROW {
|
|
break
|
|
}
|
|
|
|
rec := models.MemoryRecord{
|
|
ID: C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 0)))),
|
|
AgentID: C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 1)))),
|
|
Namespace: C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 2)))),
|
|
Content: C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 3)))),
|
|
Category: C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 4)))),
|
|
RecallCount: int(C.sqlite3_column_int64(stmt, 5)),
|
|
QualityScore: float64(C.sqlite3_column_double(stmt, 6)),
|
|
Importance: float64(C.sqlite3_column_double(stmt, 7)),
|
|
Tier: C.GoString((*C.char)(unsafe.Pointer(C.sqlite3_column_text(stmt, 8)))),
|
|
}
|
|
if lastRecalled := C.sqlite3_column_text(stmt, 9); lastRecalled != nil {
|
|
rec.LastRecalledAt, _ = time.Parse(time.RFC3339, C.GoString((*C.char)(unsafe.Pointer(lastRecalled))))
|
|
}
|
|
if createdAt := C.sqlite3_column_text(stmt, 10); createdAt != nil {
|
|
rec.CreatedAt, _ = time.Parse(time.RFC3339, C.GoString((*C.char)(unsafe.Pointer(createdAt))))
|
|
}
|
|
if updatedAt := C.sqlite3_column_text(stmt, 11); updatedAt != nil {
|
|
rec.UpdatedAt, _ = time.Parse(time.RFC3339, C.GoString((*C.char)(unsafe.Pointer(updatedAt))))
|
|
}
|
|
results = append(results, rec)
|
|
}
|
|
|
|
return results, nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) Backup(path string) error {
|
|
sc.mu.RLock()
|
|
defer sc.mu.RUnlock()
|
|
|
|
// SQLite 在线备份 API
|
|
var pBackup *C.sqlite3_backup
|
|
cDestPath := C.CString(path)
|
|
defer C.free(unsafe.Pointer(cDestPath))
|
|
|
|
var destDB *C.sqlite3
|
|
rc := C.sqlite3_open(cDestPath, &destDB)
|
|
if rc != C.SQLITE_OK {
|
|
return fmt.Errorf("backup open dest: %s", C.GoString(C.sqlite3_errmsg(destDB)))
|
|
}
|
|
defer C.sqlite3_close(destDB)
|
|
|
|
pBackup = C.sqlite3_backup_init(destDB, C.CString("main"), sc.db, C.CString("main"))
|
|
if pBackup == nil {
|
|
return fmt.Errorf("backup init: %s", C.GoString(C.sqlite3_errmsg(destDB)))
|
|
}
|
|
C.sqlite3_backup_step(pBackup, -1)
|
|
C.sqlite3_backup_finish(pBackup)
|
|
return nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) GetAuditLog(limit int) ([]map[string]interface{}, error) {
|
|
sc.mu.RLock()
|
|
defer sc.mu.RUnlock()
|
|
|
|
rows, err := sc.query(
|
|
"SELECT id,original_id,reason,deleted_at FROM tombstones ORDER BY deleted_at DESC LIMIT " + itoa(limit),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var results []map[string]interface{}
|
|
for _, r := range rows {
|
|
results = append(results, map[string]interface{}{
|
|
"id": r["id"],
|
|
"original_id": r["original_id"],
|
|
"reason": r["reason"],
|
|
"deleted_at": r["deleted_at"],
|
|
})
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) IncrementUseful(id string) error {
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
sc.execWithArgs(
|
|
"UPDATE memories SET useful_count=useful_count+1, recall_count=recall_count+1, updated_at=? WHERE id=?",
|
|
time.Now().Format(time.RFC3339), id,
|
|
)
|
|
return nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) IncrementNotUseful(id string) error {
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
sc.execWithArgs(
|
|
"UPDATE memories SET not_useful_count=not_useful_count+1, updated_at=? WHERE id=?",
|
|
time.Now().Format(time.RFC3339), id,
|
|
)
|
|
return nil
|
|
}
|
|
|
|
func (sc *SQLiteClient) UpdateMemoryContent(id, newContent, source string) error {
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
|
|
now := time.Now().Format(time.RFC3339)
|
|
|
|
// 记录版本历史
|
|
rows, _ := sc.query("SELECT version FROM memories WHERE id='" + sanitizeIdent(id) + "'")
|
|
version := int64(1)
|
|
if len(rows) > 0 {
|
|
if v, ok := rows[0]["version"].(int64); ok {
|
|
version = v + 1
|
|
}
|
|
}
|
|
|
|
sc.execWithArgs(
|
|
"INSERT INTO version_history(memory_id,version,content,source,reason,timestamp) VALUES(?,?,?,?,?,?)",
|
|
id, version, newContent, source, "corrected", now,
|
|
)
|
|
|
|
return sc.execWithArgs(
|
|
"UPDATE memories SET content=?, source=?, version=?, updated_at=? WHERE id=?",
|
|
newContent, source, version, now, id,
|
|
)
|
|
}
|
|
|
|
func (sc *SQLiteClient) Update(table, id string, fields map[string]any) error {
|
|
sc.mu.Lock()
|
|
defer sc.mu.Unlock()
|
|
|
|
if table != "memories" {
|
|
return nil
|
|
}
|
|
|
|
now := time.Now().Format(time.RFC3339)
|
|
if _, ok := fields["recall_count"]; ok {
|
|
return sc.execWithArgs(
|
|
"UPDATE memories SET recall_count=recall_count+1, updated_at=? WHERE id=?",
|
|
now, id,
|
|
)
|
|
}
|
|
if _, ok := fields["useful_count"]; ok {
|
|
return sc.execWithArgs(
|
|
"UPDATE memories SET useful_count=useful_count+1, updated_at=? WHERE id=?",
|
|
now, id,
|
|
)
|
|
}
|
|
// generic update
|
|
for k, v := range fields {
|
|
if err := sc.execWithArgs(
|
|
"UPDATE memories SET "+sanitizeIdent(k)+"=?, updated_at=? WHERE id=?",
|
|
v, now, id,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ─── helpers ──────────────────────────────────────────
|
|
|
|
func strVal(row sqliteRow, key string) string {
|
|
if v, ok := row[key]; ok {
|
|
if s, ok := v.(string); ok {
|
|
return s
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (sc *SQLiteClient) rowsToMemories(rows []sqliteRow) []models.MemoryRecord {
|
|
results := make([]models.MemoryRecord, 0, len(rows))
|
|
for _, row := range rows {
|
|
createdAt, _ := time.Parse(time.RFC3339, strVal(row, "created_at"))
|
|
results = append(results, models.MemoryRecord{
|
|
ID: strVal(row, "id"),
|
|
Content: strVal(row, "content"),
|
|
Category: strVal(row, "category"),
|
|
Namespace: strVal(row, "namespace"),
|
|
CreatedAt: createdAt,
|
|
})
|
|
}
|
|
return results
|
|
}
|
|
|
|
func itoa(n int) string {
|
|
if n <= 0 {
|
|
return "10"
|
|
}
|
|
return fmt.Sprintf("%d", n)
|
|
}
|
|
|
|
func sanitizeIdent(s string) string {
|
|
// 简单防注入:仅允许字母、数字、下划线
|
|
for _, c := range s {
|
|
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-') {
|
|
return "'invalid'"
|
|
}
|
|
}
|
|
return s
|
|
}
|