Files
memoryweave/go/internal/storage/memvector.go
T
小唯 a293f7b784 feat(storage): A10 拆出人工有用 human_useful_count 单独列(t_d618dc7c)
- Rust lance schema 追加 human_useful_count (Int64, nullable);open() 自动加列
  迁移,存量回填 NULL(与 confidence/human_not_useful_count 迁移模式一致)
- MemoryRecord 读侧接线(get_by_id/search/scan_all/scan_deleted/record_from_batch)
  走 col_i64_opt(NULL 读回 None,区别于 0)
- insert_batch 写入侧:未声明/NULL → 写 NULL;显式整数 → 写值(与 confidence 同款)
- update() $inc 基数改 col_i64_opt(NULL 行按 0 起算,首次人工 useful NULL→1)
- Go IncrementUseful 同一次 Update 三 $inc:useful_count + recall_verified_count
  + human_useful_count(只有人工 feedback 路径调用,机器自评路径不写此列)
- Go MemoryRecord/memEntry 加 *int 字段(nil=NULL)
- 新增 human_useful_count_test.go:人工 useful 同时 +1、机器路径不动反例
- 排序公式/权重零改动(AC#6 排序零变化)
2026-10-19 19:42:29 +08:00

510 lines
14 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 织忆 MemoryWeave — 内存向量存储(LanceDB 零外部依赖实现)
package storage
import (
"fmt"
"math"
"sort"
"strings"
"sync"
"time"
"github.com/xiaoxue/memoryweave/internal/models"
)
// MemLanceClient 内存实现 LanceDB 接口,生产部署时替换为 Rust LanceDB
type MemLanceClient struct {
mu sync.RWMutex
memories map[string]*memEntry
episodes []models.EpisodeRecord
auditLog []map[string]interface{}
hitEvents []models.HitEvent // GA-P0-4 (t_bb10e8bc):append-only outcome 事件
embed *Embedder
}
type memEntry struct {
ID string
Content string
Vector []float32
Category string
Namespace string
AgentID string
Tier string
QualityScore float64
UsefulCount int
NotUsefulCount int
// GA-P0-1 (t_718d8707):拆分字段
HumanNotUsefulCount int
DedupDownweightCount int
// A10 (t_d618dc7c):人工有用单独计数(指针:nil=NULL=从未被人工点过)
HumanUsefulCount *int
RecallCount int
// GA-P0-2 (t_c6d41ac6):拆分字段
RecallSeenCount int
RecallVerifiedCount int
Version int
VersionHistory []map[string]interface{}
Source string
IsDeleted bool
DeleteReason string // GA-P0-3
LastRecalledAt string
CreatedAt time.Time
UpdatedAt string
}
func NewMemLanceClient(embedder *Embedder) *MemLanceClient {
return &MemLanceClient{
memories: make(map[string]*memEntry),
embed: embedder,
}
}
// ─── LanceDB 接口实现 ─────────────────
func (mlc *MemLanceClient) InsertEpisode(agentID, namespace, content, category string) (string, error) {
id := fmt.Sprintf("ep_%d", time.Now().UnixNano())
mlc.mu.Lock()
mlc.episodes = append(mlc.episodes, models.EpisodeRecord{
ID: id,
AgentID: agentID,
Namespace: namespace,
Content: content,
Category: category,
CreatedAt: time.Now(),
})
mlc.mu.Unlock()
return id, nil
}
func (mlc *MemLanceClient) InsertMemory(m models.MemoryRecord) error {
vec, _ := mlc.embed.EncodeSingle(m.Content)
if vec == nil {
vec = make([]float32, 1024)
}
mlc.mu.Lock()
mlc.memories[m.ID] = &memEntry{
ID: m.ID,
Content: m.Content,
Vector: vec,
Category: m.Category,
Namespace: m.Namespace,
AgentID: m.AgentID,
Tier: m.Tier,
Version: 1,
CreatedAt: time.Now(),
}
mlc.mu.Unlock()
return nil
}
func (mlc *MemLanceClient) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) {
mlc.mu.RLock()
defer mlc.mu.RUnlock()
type scored struct {
entry models.MemoryRecord
score float64
}
var candidates []scored
for _, e := range mlc.memories {
if e.IsDeleted {
continue
}
if namespaceFilter != "" && e.Namespace != namespaceFilter {
continue
}
sim := cosineSim(vector, e.Vector)
candidates = append(candidates, scored{
entry: models.MemoryRecord{
ID: e.ID,
Content: e.Content,
Category: e.Category,
},
score: sim,
})
}
sort.Slice(candidates, func(i, j int) bool {
return candidates[i].score > candidates[j].score
})
if topK > len(candidates) {
topK = len(candidates)
}
results := make([]models.MemoryRecord, topK)
for i := 0; i < topK; i++ {
results[i] = candidates[i].entry
}
return results, nil
}
func (mlc *MemLanceClient) Insert(table string, record any) error {
if r, ok := record.(models.MemoryRecord); ok {
return mlc.InsertMemory(r)
}
if r, ok := record.(models.EpisodeRecord); ok {
mlc.mu.Lock()
mlc.episodes = append(mlc.episodes, r)
mlc.mu.Unlock()
return nil
}
return fmt.Errorf("unknown record type")
}
func (mlc *MemLanceClient) GetTopByQuality(agentID string, limit int) ([]models.MemoryRecord, error) {
mlc.mu.RLock()
defer mlc.mu.RUnlock()
var results []models.MemoryRecord
for _, e := range mlc.memories {
if e.IsDeleted {
continue
}
results = append(results, models.MemoryRecord{
ID: e.ID,
Content: e.Content,
Category: e.Category,
Namespace: e.Namespace,
})
if len(results) >= limit {
break
}
}
return results, nil
}
func (mlc *MemLanceClient) Stats() (map[string]interface{}, error) {
mlc.mu.RLock()
defer mlc.mu.RUnlock()
return map[string]interface{}{
"memory_count": len(mlc.memories),
"episode_count": len(mlc.episodes),
"backend": "in-memory (zero-deps)",
}, nil
}
func (mlc *MemLanceClient) SoftDelete(id, reason string) error {
mlc.mu.Lock()
defer mlc.mu.Unlock()
if e, ok := mlc.memories[id]; ok {
e.IsDeleted = true
e.DeleteReason = NormalizeDeleteReason(reason) // GA-P0-3
}
return nil
}
func (mlc *MemLanceClient) GetVersionHistory(id string) ([]map[string]interface{}, error) {
mlc.mu.RLock()
defer mlc.mu.RUnlock()
if e, ok := mlc.memories[id]; ok {
return e.VersionHistory, nil
}
return nil, nil
}
func (mlc *MemLanceClient) GetCandidatesForForgetting() ([]map[string]interface{}, error) {
mlc.mu.RLock()
defer mlc.mu.RUnlock()
var results []map[string]interface{}
for _, e := range mlc.memories {
if !e.IsDeleted && e.Tier != "core" {
results = append(results, map[string]interface{}{
"id": e.ID,
})
}
}
return results, nil
}
// Scan 内存实现的全表扫描(过滤 is_deleted + namespace)。
// 按 CreatedAt 倒序返回(列表/导出更符合直觉:新的在前)。
func (mlc *MemLanceClient) Scan(limit int, namespaceFilter string) ([]models.MemoryRecord, error) {
if limit <= 0 {
limit = 100
}
mlc.mu.RLock()
defer mlc.mu.RUnlock()
type item struct {
rec models.MemoryRecord
ts time.Time
}
items := make([]item, 0, len(mlc.memories))
for _, e := range mlc.memories {
if e.IsDeleted {
continue
}
if namespaceFilter != "" && e.Namespace != namespaceFilter {
continue
}
items = append(items, item{
rec: models.MemoryRecord{
ID: e.ID, Content: e.Content, Category: e.Category,
Namespace: e.Namespace, CreatedAt: e.CreatedAt,
QualityScore: e.QualityScore, RecallCount: e.RecallCount,
Tier: e.Tier, Source: e.Source, Vector: e.Vector,
},
ts: e.CreatedAt,
})
}
sort.Slice(items, func(i, j int) bool { return items[i].ts.After(items[j].ts) })
out := make([]models.MemoryRecord, 0, limit)
for _, it := range items {
if len(out) >= limit {
break
}
out = append(out, it.rec)
}
return out, nil
}
func (mlc *MemLanceClient) GetSkillCandidates(minRecalls, limit int) ([]models.MemoryRecord, error) {
return nil, nil // 内存实现,暂不支持
}
func (mlc *MemLanceClient) Backup(path string) error {
return nil // 内存实现无需备份
}
func (mlc *MemLanceClient) GetAuditLog(limit int) ([]map[string]interface{}, error) {
mlc.mu.RLock()
defer mlc.mu.RUnlock()
if limit > len(mlc.auditLog) {
limit = len(mlc.auditLog)
}
return mlc.auditLog[:limit], nil
}
func (mlc *MemLanceClient) IncrementUseful(id string) error {
mlc.mu.Lock()
defer mlc.mu.Unlock()
if e, ok := mlc.memories[id]; ok {
e.UsefulCount++
// GA-P0-2 (t_c6d41ac6):正反馈 → recall_verified_count++;不再动 recall_count(冻结)
e.RecallVerifiedCount++
// A10 (t_d618dc7c):人工 useful ⇒ human_useful_count 同步 +1(与生产 IncrementUseful 的
// useful_count+recall_verified_count+human_useful_count 三 $inc 同一次 Update 一致)。
// 机器自评/蒸馏路径(IncrementDedupDownweight / IncrementHumanNotUseful)不碰此列。
if e.HumanUsefulCount == nil {
e.HumanUsefulCount = new(int)
}
*e.HumanUsefulCount++
}
return nil
}
// IncrementNotUseful [DEPRECATED](GA-P0-1 t_718d8707)历史混写列;新代码走
// IncrementHumanNotUseful(人工) / IncrementDedupDownweight(dedup 降权)。
func (mlc *MemLanceClient) IncrementNotUseful(id string) error {
mlc.mu.Lock()
defer mlc.mu.Unlock()
if e, ok := mlc.memories[id]; ok {
e.NotUsefulCount++
}
return nil
}
// IncrementHumanNotUseful 递增 human_not_useful_count(用户人工点「没用」,GA-P0-1)。
func (mlc *MemLanceClient) IncrementHumanNotUseful(id string) error {
mlc.mu.Lock()
defer mlc.mu.Unlock()
if e, ok := mlc.memories[id]; ok {
e.HumanNotUsefulCount++
}
return nil
}
// IncrementDedupDownweight 递增 dedup_downweight_count(dedup/合并流程自动降权,GA-P0-1)。
func (mlc *MemLanceClient) IncrementDedupDownweight(id string) error {
mlc.mu.Lock()
defer mlc.mu.Unlock()
if e, ok := mlc.memories[id]; ok {
e.DedupDownweightCount++
}
return nil
}
// AppendHitEvent 追加一条真实 outcome 事件(GA-P0-4 t_bb10e8bc,内存实现)。
func (mlc *MemLanceClient) AppendHitEvent(memoryID, outcome, reason string) error {
outcome = NormalizeHitOutcome(outcome)
if !ValidHitOutcomes[outcome] {
return fmt.Errorf("AppendHitEvent: invalid outcome %q (must be pos/neg); dedup-demote/recall-appearance are NOT outcomes", outcome)
}
reason = NormalizeHitReason(reason)
mlc.mu.Lock()
defer mlc.mu.Unlock()
mlc.hitEvents = append(mlc.hitEvents, models.HitEvent{
ID: int64(len(mlc.hitEvents) + 1),
MemoryID: memoryID,
Outcome: outcome,
Reason: reason,
Source: "append_hit_event",
CreatedAt: time.Now(),
})
return nil
}
// CountHitEvents 统计某 memory 在某时间窗起点之后的 outcome 事件数(内存实现)。
func (mlc *MemLanceClient) CountHitEvents(memoryID, outcome string, since time.Time) (int, error) {
outcome = NormalizeHitOutcome(outcome)
if !ValidHitOutcomes[outcome] {
return 0, fmt.Errorf("CountHitEvents: invalid outcome %q", outcome)
}
mlc.mu.RLock()
defer mlc.mu.RUnlock()
n := 0
for _, ev := range mlc.hitEvents {
if ev.MemoryID == memoryID && ev.Outcome == outcome && !ev.CreatedAt.Before(since) {
n++
}
}
return n, nil
}
// GetHitEvents 读回某 memory 的事件明细(内存实现)。
func (mlc *MemLanceClient) GetHitEvents(memoryID string, limit int) ([]models.HitEvent, error) {
mlc.mu.RLock()
defer mlc.mu.RUnlock()
if limit <= 0 {
limit = 100
}
var out []models.HitEvent
for _, ev := range mlc.hitEvents {
if ev.MemoryID == memoryID {
out = append(out, ev)
if len(out) >= limit {
break
}
}
}
return out, nil
}
func (mlc *MemLanceClient) UpdateMemoryContent(id, newContent, source string) error {
mlc.mu.Lock()
defer mlc.mu.Unlock()
if e, ok := mlc.memories[id]; ok {
e.Version++
e.VersionHistory = append(e.VersionHistory, map[string]interface{}{
"version": e.Version,
"content": newContent,
"source": source,
"reason": "corrected",
"timestamp": time.Now().Format(time.RFC3339),
})
e.Content = newContent
e.Source = source
// 重新编码
if vec, err := mlc.embed.EncodeSingle(newContent); err == nil {
e.Vector = vec
}
}
return nil
}
// Update 更新记录字段
func (mlc *MemLanceClient) Update(table, id string, fields map[string]any) error {
mlc.mu.Lock()
defer mlc.mu.Unlock()
if table == "memories" {
if e, ok := mlc.memories[id]; ok {
// GA-P0-2 (t_c6d41ac6):recall_count 冻结。召回命中改走 recall_seen_count;
// 正反馈走 recall_verified_count(也经 Update 路径的反馈端点)。
if v, ok := fields["recall_seen_count"]; ok {
if incMap, ok := v.(map[string]string); ok {
if incStr, ok := incMap["$inc"]; ok {
if incStr == "1" {
e.RecallSeenCount++
}
}
}
}
if v, ok := fields["recall_verified_count"]; ok {
if incMap, ok := v.(map[string]string); ok {
if incStr, ok := incMap["$inc"]; ok {
if incStr == "1" {
e.RecallVerifiedCount++
}
}
}
}
if v, ok := fields["recall_count"]; ok {
// handle $inc
if incMap, ok := v.(map[string]string); ok {
if incStr, ok := incMap["$inc"]; ok {
if incStr == "1" {
e.RecallCount++
}
}
}
}
}
}
return nil
}
// PurgeInvalid 清理 id 为空的垃圾行(P1-1 2026-10-08,内存兜底实现)。
func (mlc *MemLanceClient) PurgeInvalid() (int64, error) {
mlc.mu.Lock()
defer mlc.mu.Unlock()
var n int64
for id, e := range mlc.memories {
if strings.TrimSpace(id) == "" && !e.IsDeleted {
e.IsDeleted = true
e.DeleteReason = DeleteReasonRetention // GA-P0-3
n++
}
}
return n, nil
}
// ─── SearchCache 兼容 ──────────────────────────────────
func (mlc *MemLanceClient) SearchCompat(queryVec []float32, topK int, namespace string) ([]interface{}, error) {
records, err := mlc.Search("memories", queryVec, topK, namespace)
if err != nil {
return nil, err
}
results := make([]interface{}, len(records))
for i, r := range records {
results[i] = r
}
return results, nil
}
// ─── 工具 ─────────────────────────────────────────────
func cosineSim(a, b []float32) float64 {
if len(a) != len(b) || len(a) == 0 {
return 0
}
var dot, normA, normB float64
for i := range a {
dot += float64(a[i]) * float64(b[i])
normA += float64(a[i]) * float64(a[i])
normB += float64(b[i]) * float64(b[i])
}
if normA == 0 || normB == 0 {
return 0
}
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
}
// SoftDeleteBatch 批量标记 is_deleted=true,updated_at=ts(单事务提交)
func (mlc *MemLanceClient) SoftDeleteBatch(table string, ids []string, updatedAt string) (int64, error) {
if table != "memories" {
return 0, fmt.Errorf("unsupported table: %s", table)
}
mlc.mu.Lock()
defer mlc.mu.Unlock()
var updated int64
for _, id := range ids {
if e, ok := mlc.memories[id]; ok {
if !e.IsDeleted {
e.IsDeleted = true
e.DeleteReason = DeleteReasonUnknown // GA-P0-3:批量无逐条原因
e.UpdatedAt = updatedAt
updated++
}
}
}
return updated, nil
}