- Rust lance schema 追加 human_useful_count (Int64, nullable);open() 自动加列 迁移,存量回填 NULL(与 confidence/human_not_useful_count 迁移模式一致) - MemoryRecord 读侧接线(get_by_id/search/scan_all/scan_deleted/record_from_batch) 走 col_i64_opt(NULL 读回 None,区别于 0) - insert_batch 写入侧:未声明/NULL → 写 NULL;显式整数 → 写值(与 confidence 同款) - update() $inc 基数改 col_i64_opt(NULL 行按 0 起算,首次人工 useful NULL→1) - Go IncrementUseful 同一次 Update 三 $inc:useful_count + recall_verified_count + human_useful_count(只有人工 feedback 路径调用,机器自评路径不写此列) - Go MemoryRecord/memEntry 加 *int 字段(nil=NULL) - 新增 human_useful_count_test.go:人工 useful 同时 +1、机器路径不动反例 - 排序公式/权重零改动(AC#6 排序零变化)
510 lines
14 KiB
Go
510 lines
14 KiB
Go
// 织忆 MemoryWeave — 内存向量存储(LanceDB 零外部依赖实现)
|
||
package storage
|
||
|
||
import (
|
||
"fmt"
|
||
"math"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/xiaoxue/memoryweave/internal/models"
|
||
)
|
||
|
||
// MemLanceClient 内存实现 LanceDB 接口,生产部署时替换为 Rust LanceDB
|
||
type MemLanceClient struct {
|
||
mu sync.RWMutex
|
||
memories map[string]*memEntry
|
||
episodes []models.EpisodeRecord
|
||
auditLog []map[string]interface{}
|
||
hitEvents []models.HitEvent // GA-P0-4 (t_bb10e8bc):append-only outcome 事件
|
||
embed *Embedder
|
||
}
|
||
|
||
type memEntry struct {
|
||
ID string
|
||
Content string
|
||
Vector []float32
|
||
Category string
|
||
Namespace string
|
||
AgentID string
|
||
Tier string
|
||
QualityScore float64
|
||
UsefulCount int
|
||
NotUsefulCount int
|
||
// GA-P0-1 (t_718d8707):拆分字段
|
||
HumanNotUsefulCount int
|
||
DedupDownweightCount int
|
||
// A10 (t_d618dc7c):人工有用单独计数(指针:nil=NULL=从未被人工点过)
|
||
HumanUsefulCount *int
|
||
RecallCount int
|
||
// GA-P0-2 (t_c6d41ac6):拆分字段
|
||
RecallSeenCount int
|
||
RecallVerifiedCount int
|
||
Version int
|
||
VersionHistory []map[string]interface{}
|
||
Source string
|
||
IsDeleted bool
|
||
DeleteReason string // GA-P0-3
|
||
LastRecalledAt string
|
||
CreatedAt time.Time
|
||
UpdatedAt string
|
||
}
|
||
|
||
func NewMemLanceClient(embedder *Embedder) *MemLanceClient {
|
||
return &MemLanceClient{
|
||
memories: make(map[string]*memEntry),
|
||
embed: embedder,
|
||
}
|
||
}
|
||
|
||
// ─── LanceDB 接口实现 ─────────────────
|
||
|
||
func (mlc *MemLanceClient) InsertEpisode(agentID, namespace, content, category string) (string, error) {
|
||
id := fmt.Sprintf("ep_%d", time.Now().UnixNano())
|
||
mlc.mu.Lock()
|
||
mlc.episodes = append(mlc.episodes, models.EpisodeRecord{
|
||
ID: id,
|
||
AgentID: agentID,
|
||
Namespace: namespace,
|
||
Content: content,
|
||
Category: category,
|
||
CreatedAt: time.Now(),
|
||
})
|
||
mlc.mu.Unlock()
|
||
return id, nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) InsertMemory(m models.MemoryRecord) error {
|
||
vec, _ := mlc.embed.EncodeSingle(m.Content)
|
||
if vec == nil {
|
||
vec = make([]float32, 1024)
|
||
}
|
||
mlc.mu.Lock()
|
||
mlc.memories[m.ID] = &memEntry{
|
||
ID: m.ID,
|
||
Content: m.Content,
|
||
Vector: vec,
|
||
Category: m.Category,
|
||
Namespace: m.Namespace,
|
||
AgentID: m.AgentID,
|
||
Tier: m.Tier,
|
||
Version: 1,
|
||
CreatedAt: time.Now(),
|
||
}
|
||
mlc.mu.Unlock()
|
||
return nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
||
mlc.mu.RLock()
|
||
defer mlc.mu.RUnlock()
|
||
|
||
type scored struct {
|
||
entry models.MemoryRecord
|
||
score float64
|
||
}
|
||
var candidates []scored
|
||
for _, e := range mlc.memories {
|
||
if e.IsDeleted {
|
||
continue
|
||
}
|
||
if namespaceFilter != "" && e.Namespace != namespaceFilter {
|
||
continue
|
||
}
|
||
sim := cosineSim(vector, e.Vector)
|
||
candidates = append(candidates, scored{
|
||
entry: models.MemoryRecord{
|
||
ID: e.ID,
|
||
Content: e.Content,
|
||
Category: e.Category,
|
||
},
|
||
score: sim,
|
||
})
|
||
}
|
||
sort.Slice(candidates, func(i, j int) bool {
|
||
return candidates[i].score > candidates[j].score
|
||
})
|
||
if topK > len(candidates) {
|
||
topK = len(candidates)
|
||
}
|
||
results := make([]models.MemoryRecord, topK)
|
||
for i := 0; i < topK; i++ {
|
||
results[i] = candidates[i].entry
|
||
}
|
||
return results, nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) Insert(table string, record any) error {
|
||
if r, ok := record.(models.MemoryRecord); ok {
|
||
return mlc.InsertMemory(r)
|
||
}
|
||
if r, ok := record.(models.EpisodeRecord); ok {
|
||
mlc.mu.Lock()
|
||
mlc.episodes = append(mlc.episodes, r)
|
||
mlc.mu.Unlock()
|
||
return nil
|
||
}
|
||
return fmt.Errorf("unknown record type")
|
||
}
|
||
|
||
func (mlc *MemLanceClient) GetTopByQuality(agentID string, limit int) ([]models.MemoryRecord, error) {
|
||
mlc.mu.RLock()
|
||
defer mlc.mu.RUnlock()
|
||
|
||
var results []models.MemoryRecord
|
||
for _, e := range mlc.memories {
|
||
if e.IsDeleted {
|
||
continue
|
||
}
|
||
results = append(results, models.MemoryRecord{
|
||
ID: e.ID,
|
||
Content: e.Content,
|
||
Category: e.Category,
|
||
Namespace: e.Namespace,
|
||
})
|
||
if len(results) >= limit {
|
||
break
|
||
}
|
||
}
|
||
return results, nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) Stats() (map[string]interface{}, error) {
|
||
mlc.mu.RLock()
|
||
defer mlc.mu.RUnlock()
|
||
return map[string]interface{}{
|
||
"memory_count": len(mlc.memories),
|
||
"episode_count": len(mlc.episodes),
|
||
"backend": "in-memory (zero-deps)",
|
||
}, nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) SoftDelete(id, reason string) error {
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
if e, ok := mlc.memories[id]; ok {
|
||
e.IsDeleted = true
|
||
e.DeleteReason = NormalizeDeleteReason(reason) // GA-P0-3
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) GetVersionHistory(id string) ([]map[string]interface{}, error) {
|
||
mlc.mu.RLock()
|
||
defer mlc.mu.RUnlock()
|
||
if e, ok := mlc.memories[id]; ok {
|
||
return e.VersionHistory, nil
|
||
}
|
||
return nil, nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) GetCandidatesForForgetting() ([]map[string]interface{}, error) {
|
||
mlc.mu.RLock()
|
||
defer mlc.mu.RUnlock()
|
||
var results []map[string]interface{}
|
||
for _, e := range mlc.memories {
|
||
if !e.IsDeleted && e.Tier != "core" {
|
||
results = append(results, map[string]interface{}{
|
||
"id": e.ID,
|
||
})
|
||
}
|
||
}
|
||
return results, nil
|
||
}
|
||
|
||
// Scan 内存实现的全表扫描(过滤 is_deleted + namespace)。
|
||
// 按 CreatedAt 倒序返回(列表/导出更符合直觉:新的在前)。
|
||
func (mlc *MemLanceClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
||
if limit <= 0 {
|
||
limit = 100
|
||
}
|
||
mlc.mu.RLock()
|
||
defer mlc.mu.RUnlock()
|
||
type item struct {
|
||
rec models.MemoryRecord
|
||
ts time.Time
|
||
}
|
||
items := make([]item, 0, len(mlc.memories))
|
||
for _, e := range mlc.memories {
|
||
if e.IsDeleted {
|
||
continue
|
||
}
|
||
if namespaceFilter != "" && e.Namespace != namespaceFilter {
|
||
continue
|
||
}
|
||
items = append(items, item{
|
||
rec: models.MemoryRecord{
|
||
ID: e.ID, Content: e.Content, Category: e.Category,
|
||
Namespace: e.Namespace, CreatedAt: e.CreatedAt,
|
||
QualityScore: e.QualityScore, RecallCount: e.RecallCount,
|
||
Tier: e.Tier, Source: e.Source, Vector: e.Vector,
|
||
},
|
||
ts: e.CreatedAt,
|
||
})
|
||
}
|
||
sort.Slice(items, func(i, j int) bool { return items[i].ts.After(items[j].ts) })
|
||
out := make([]models.MemoryRecord, 0, limit)
|
||
for _, it := range items {
|
||
if len(out) >= limit {
|
||
break
|
||
}
|
||
out = append(out, it.rec)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) GetSkillCandidates(minRecalls, limit int) ([]models.MemoryRecord, error) {
|
||
return nil, nil // 内存实现,暂不支持
|
||
}
|
||
|
||
func (mlc *MemLanceClient) Backup(path string) error {
|
||
return nil // 内存实现无需备份
|
||
}
|
||
|
||
func (mlc *MemLanceClient) GetAuditLog(limit int) ([]map[string]interface{}, error) {
|
||
mlc.mu.RLock()
|
||
defer mlc.mu.RUnlock()
|
||
if limit > len(mlc.auditLog) {
|
||
limit = len(mlc.auditLog)
|
||
}
|
||
return mlc.auditLog[:limit], nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) IncrementUseful(id string) error {
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
if e, ok := mlc.memories[id]; ok {
|
||
e.UsefulCount++
|
||
// GA-P0-2 (t_c6d41ac6):正反馈 → recall_verified_count++;不再动 recall_count(冻结)
|
||
e.RecallVerifiedCount++
|
||
// A10 (t_d618dc7c):人工 useful ⇒ human_useful_count 同步 +1(与生产 IncrementUseful 的
|
||
// useful_count+recall_verified_count+human_useful_count 三 $inc 同一次 Update 一致)。
|
||
// 机器自评/蒸馏路径(IncrementDedupDownweight / IncrementHumanNotUseful)不碰此列。
|
||
if e.HumanUsefulCount == nil {
|
||
e.HumanUsefulCount = new(int)
|
||
}
|
||
*e.HumanUsefulCount++
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// IncrementNotUseful [DEPRECATED](GA-P0-1 t_718d8707)历史混写列;新代码走
|
||
// IncrementHumanNotUseful(人工) / IncrementDedupDownweight(dedup 降权)。
|
||
func (mlc *MemLanceClient) IncrementNotUseful(id string) error {
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
if e, ok := mlc.memories[id]; ok {
|
||
e.NotUsefulCount++
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// IncrementHumanNotUseful 递增 human_not_useful_count(用户人工点「没用」,GA-P0-1)。
|
||
func (mlc *MemLanceClient) IncrementHumanNotUseful(id string) error {
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
if e, ok := mlc.memories[id]; ok {
|
||
e.HumanNotUsefulCount++
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// IncrementDedupDownweight 递增 dedup_downweight_count(dedup/合并流程自动降权,GA-P0-1)。
|
||
func (mlc *MemLanceClient) IncrementDedupDownweight(id string) error {
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
if e, ok := mlc.memories[id]; ok {
|
||
e.DedupDownweightCount++
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// AppendHitEvent 追加一条真实 outcome 事件(GA-P0-4 t_bb10e8bc,内存实现)。
|
||
func (mlc *MemLanceClient) AppendHitEvent(memoryID, outcome, reason string) error {
|
||
outcome = NormalizeHitOutcome(outcome)
|
||
if !ValidHitOutcomes[outcome] {
|
||
return fmt.Errorf("AppendHitEvent: invalid outcome %q (must be pos/neg); dedup-demote/recall-appearance are NOT outcomes", outcome)
|
||
}
|
||
reason = NormalizeHitReason(reason)
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
mlc.hitEvents = append(mlc.hitEvents, models.HitEvent{
|
||
ID: int64(len(mlc.hitEvents) + 1),
|
||
MemoryID: memoryID,
|
||
Outcome: outcome,
|
||
Reason: reason,
|
||
Source: "append_hit_event",
|
||
CreatedAt: time.Now(),
|
||
})
|
||
return nil
|
||
}
|
||
|
||
// CountHitEvents 统计某 memory 在某时间窗起点之后的 outcome 事件数(内存实现)。
|
||
func (mlc *MemLanceClient) CountHitEvents(memoryID, outcome string, since time.Time) (int, error) {
|
||
outcome = NormalizeHitOutcome(outcome)
|
||
if !ValidHitOutcomes[outcome] {
|
||
return 0, fmt.Errorf("CountHitEvents: invalid outcome %q", outcome)
|
||
}
|
||
mlc.mu.RLock()
|
||
defer mlc.mu.RUnlock()
|
||
n := 0
|
||
for _, ev := range mlc.hitEvents {
|
||
if ev.MemoryID == memoryID && ev.Outcome == outcome && !ev.CreatedAt.Before(since) {
|
||
n++
|
||
}
|
||
}
|
||
return n, nil
|
||
}
|
||
|
||
// GetHitEvents 读回某 memory 的事件明细(内存实现)。
|
||
func (mlc *MemLanceClient) GetHitEvents(memoryID string, limit int) ([]models.HitEvent, error) {
|
||
mlc.mu.RLock()
|
||
defer mlc.mu.RUnlock()
|
||
if limit <= 0 {
|
||
limit = 100
|
||
}
|
||
var out []models.HitEvent
|
||
for _, ev := range mlc.hitEvents {
|
||
if ev.MemoryID == memoryID {
|
||
out = append(out, ev)
|
||
if len(out) >= limit {
|
||
break
|
||
}
|
||
}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (mlc *MemLanceClient) UpdateMemoryContent(id, newContent, source string) error {
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
if e, ok := mlc.memories[id]; ok {
|
||
e.Version++
|
||
e.VersionHistory = append(e.VersionHistory, map[string]interface{}{
|
||
"version": e.Version,
|
||
"content": newContent,
|
||
"source": source,
|
||
"reason": "corrected",
|
||
"timestamp": time.Now().Format(time.RFC3339),
|
||
})
|
||
e.Content = newContent
|
||
e.Source = source
|
||
// 重新编码
|
||
if vec, err := mlc.embed.EncodeSingle(newContent); err == nil {
|
||
e.Vector = vec
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Update 更新记录字段
|
||
func (mlc *MemLanceClient) Update(table, id string, fields map[string]any) error {
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
if table == "memories" {
|
||
if e, ok := mlc.memories[id]; ok {
|
||
// GA-P0-2 (t_c6d41ac6):recall_count 冻结。召回命中改走 recall_seen_count;
|
||
// 正反馈走 recall_verified_count(也经 Update 路径的反馈端点)。
|
||
if v, ok := fields["recall_seen_count"]; ok {
|
||
if incMap, ok := v.(map[string]string); ok {
|
||
if incStr, ok := incMap["$inc"]; ok {
|
||
if incStr == "1" {
|
||
e.RecallSeenCount++
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if v, ok := fields["recall_verified_count"]; ok {
|
||
if incMap, ok := v.(map[string]string); ok {
|
||
if incStr, ok := incMap["$inc"]; ok {
|
||
if incStr == "1" {
|
||
e.RecallVerifiedCount++
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if v, ok := fields["recall_count"]; ok {
|
||
// handle $inc
|
||
if incMap, ok := v.(map[string]string); ok {
|
||
if incStr, ok := incMap["$inc"]; ok {
|
||
if incStr == "1" {
|
||
e.RecallCount++
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// PurgeInvalid 清理 id 为空的垃圾行(P1-1 2026-10-08,内存兜底实现)。
|
||
func (mlc *MemLanceClient) PurgeInvalid() (int64, error) {
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
var n int64
|
||
for id, e := range mlc.memories {
|
||
if strings.TrimSpace(id) == "" && !e.IsDeleted {
|
||
e.IsDeleted = true
|
||
e.DeleteReason = DeleteReasonRetention // GA-P0-3
|
||
n++
|
||
}
|
||
}
|
||
return n, nil
|
||
}
|
||
|
||
// ─── SearchCache 兼容 ──────────────────────────────────
|
||
|
||
func (mlc *MemLanceClient) SearchCompat(queryVec []float32, topK int, namespace string) ([]interface{}, error) {
|
||
records, err := mlc.Search("memories", queryVec, topK, namespace)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
results := make([]interface{}, len(records))
|
||
for i, r := range records {
|
||
results[i] = r
|
||
}
|
||
return results, nil
|
||
}
|
||
|
||
// ─── 工具 ─────────────────────────────────────────────
|
||
|
||
func cosineSim(a, b []float32) float64 {
|
||
if len(a) != len(b) || len(a) == 0 {
|
||
return 0
|
||
}
|
||
var dot, normA, normB float64
|
||
for i := range a {
|
||
dot += float64(a[i]) * float64(b[i])
|
||
normA += float64(a[i]) * float64(a[i])
|
||
normB += float64(b[i]) * float64(b[i])
|
||
}
|
||
if normA == 0 || normB == 0 {
|
||
return 0
|
||
}
|
||
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
|
||
}
|
||
|
||
// SoftDeleteBatch 批量标记 is_deleted=true,updated_at=ts(单事务提交)
|
||
func (mlc *MemLanceClient) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) {
|
||
if table != "memories" {
|
||
return 0, fmt.Errorf("unsupported table: %s", table)
|
||
}
|
||
mlc.mu.Lock()
|
||
defer mlc.mu.Unlock()
|
||
var updated int64
|
||
for _, id := range ids {
|
||
if e, ok := mlc.memories[id]; ok {
|
||
if !e.IsDeleted {
|
||
e.IsDeleted = true
|
||
e.DeleteReason = DeleteReasonUnknown // GA-P0-3:批量无逐条原因
|
||
e.UpdatedAt = updatedAt
|
||
updated++
|
||
}
|
||
}
|
||
}
|
||
return updated, nil
|
||
}
|