G7.1 — Skill 持久化:
- BetaSkill: prompt_template, linked_memory_ids, linked_entities, created_at
- BayesianSkillManager: EnableRedisPersistence(), loadFromRedis(), persistSkill()
- Redis HASH (zhiyi:skills) 持久化,重启不丢 skills
- Skill CRUD API: POST/GET/DELETE /api/v1/skills/{name}
- auto_distill skill: ETA=0.93, status=active after 12 trials ✓
G7.2 — Skill 结晶:
- GetSkillCandidates(minRecalls, limit): storage.LanceDB 接口扩展
- SQLiteClient: SQL 查询候选(recall_count>=5, quality_score>=0.7)
- RustLanceDBClient/MemLanceClient: stub 实现
- /api/v1/crystallize/candidates: 获取候选记忆
- /api/v1/crystallize/memory/{id}: 对记忆执行 LLM 结晶
- callSkillLLM(): LLM 生成 prompt 模板(带降级)
- RegisterLDBGetter(): routes 包访问 LanceDB 实例
312 lines
7.4 KiB
Go
312 lines
7.4 KiB
Go
// 织忆 MemoryWeave — 内存向量存储(LanceDB 零外部依赖实现)
|
|
package storage
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/xiaoxue/memoryweave/internal/models"
|
|
)
|
|
|
|
// MemLanceClient 内存实现 LanceDB 接口,生产部署时替换为 Rust LanceDB
|
|
type MemLanceClient struct {
|
|
mu sync.RWMutex
|
|
memories map[string]*memEntry
|
|
episodes []models.EpisodeRecord
|
|
auditLog []map[string]interface{}
|
|
embed *Embedder
|
|
}
|
|
|
|
type memEntry struct {
|
|
ID string
|
|
Content string
|
|
Vector []float32
|
|
Category string
|
|
Namespace string
|
|
AgentID string
|
|
Tier string
|
|
QualityScore float64
|
|
UsefulCount int
|
|
NotUsefulCount int
|
|
RecallCount int
|
|
Version int
|
|
VersionHistory []map[string]interface{}
|
|
Source string
|
|
IsDeleted bool
|
|
LastRecalledAt string
|
|
CreatedAt time.Time
|
|
}
|
|
|
|
func NewMemLanceClient(embedder *Embedder) *MemLanceClient {
|
|
return &MemLanceClient{
|
|
memories: make(map[string]*memEntry),
|
|
embed: embedder,
|
|
}
|
|
}
|
|
|
|
// ─── LanceDB 接口实现 ─────────────────────────────────
|
|
|
|
func (mlc *MemLanceClient) InsertEpisode(agentID, namespace, content, category string) (string, error) {
|
|
id := fmt.Sprintf("ep_%d", time.Now().UnixNano())
|
|
mlc.mu.Lock()
|
|
mlc.episodes = append(mlc.episodes, models.EpisodeRecord{
|
|
ID: id,
|
|
AgentID: agentID,
|
|
Namespace: namespace,
|
|
Content: content,
|
|
Category: category,
|
|
CreatedAt: time.Now(),
|
|
})
|
|
mlc.mu.Unlock()
|
|
return id, nil
|
|
}
|
|
|
|
func (mlc *MemLanceClient) InsertMemory(m models.MemoryRecord) error {
|
|
vec, _ := mlc.embed.EncodeSingle(m.Content)
|
|
if vec == nil {
|
|
vec = make([]float32, 1024)
|
|
}
|
|
mlc.mu.Lock()
|
|
mlc.memories[m.ID] = &memEntry{
|
|
ID: m.ID,
|
|
Content: m.Content,
|
|
Vector: vec,
|
|
Category: m.Category,
|
|
Namespace: m.Namespace,
|
|
AgentID: m.AgentID,
|
|
Tier: m.Tier,
|
|
Version: 1,
|
|
CreatedAt: time.Now(),
|
|
}
|
|
mlc.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
func (mlc *MemLanceClient) Search(table string, vector []float32, topK int, namespaceFilter string) ([]models.MemoryRecord, error) {
|
|
mlc.mu.RLock()
|
|
defer mlc.mu.RUnlock()
|
|
|
|
type scored struct {
|
|
entry models.MemoryRecord
|
|
score float64
|
|
}
|
|
var candidates []scored
|
|
for _, e := range mlc.memories {
|
|
if e.IsDeleted {
|
|
continue
|
|
}
|
|
if namespaceFilter != "" && e.Namespace != namespaceFilter {
|
|
continue
|
|
}
|
|
sim := cosineSim(vector, e.Vector)
|
|
candidates = append(candidates, scored{
|
|
entry: models.MemoryRecord{
|
|
ID: e.ID,
|
|
Content: e.Content,
|
|
Category: e.Category,
|
|
},
|
|
score: sim,
|
|
})
|
|
}
|
|
sort.Slice(candidates, func(i, j int) bool {
|
|
return candidates[i].score > candidates[j].score
|
|
})
|
|
if topK > len(candidates) {
|
|
topK = len(candidates)
|
|
}
|
|
results := make([]models.MemoryRecord, topK)
|
|
for i := 0; i < topK; i++ {
|
|
results[i] = candidates[i].entry
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (mlc *MemLanceClient) Insert(table string, record any) error {
|
|
if r, ok := record.(models.MemoryRecord); ok {
|
|
return mlc.InsertMemory(r)
|
|
}
|
|
if r, ok := record.(models.EpisodeRecord); ok {
|
|
mlc.mu.Lock()
|
|
mlc.episodes = append(mlc.episodes, r)
|
|
mlc.mu.Unlock()
|
|
return nil
|
|
}
|
|
return fmt.Errorf("unknown record type")
|
|
}
|
|
|
|
func (mlc *MemLanceClient) GetTopByQuality(agentID string, limit int) ([]models.MemoryRecord, error) {
|
|
mlc.mu.RLock()
|
|
defer mlc.mu.RUnlock()
|
|
|
|
var results []models.MemoryRecord
|
|
for _, e := range mlc.memories {
|
|
if e.IsDeleted {
|
|
continue
|
|
}
|
|
results = append(results, models.MemoryRecord{
|
|
ID: e.ID,
|
|
Content: e.Content,
|
|
Category: e.Category,
|
|
Namespace: e.Namespace,
|
|
})
|
|
if len(results) >= limit {
|
|
break
|
|
}
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (mlc *MemLanceClient) Stats() (map[string]interface{}, error) {
|
|
mlc.mu.RLock()
|
|
defer mlc.mu.RUnlock()
|
|
return map[string]interface{}{
|
|
"memory_count": len(mlc.memories),
|
|
"episode_count": len(mlc.episodes),
|
|
"backend": "in-memory (zero-deps)",
|
|
}, nil
|
|
}
|
|
|
|
func (mlc *MemLanceClient) SoftDelete(id, reason string) error {
|
|
mlc.mu.Lock()
|
|
defer mlc.mu.Unlock()
|
|
if e, ok := mlc.memories[id]; ok {
|
|
e.IsDeleted = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (mlc *MemLanceClient) GetVersionHistory(id string) ([]map[string]interface{}, error) {
|
|
mlc.mu.RLock()
|
|
defer mlc.mu.RUnlock()
|
|
if e, ok := mlc.memories[id]; ok {
|
|
return e.VersionHistory, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (mlc *MemLanceClient) GetCandidatesForForgetting() ([]map[string]interface{}, error) {
|
|
mlc.mu.RLock()
|
|
defer mlc.mu.RUnlock()
|
|
var results []map[string]interface{}
|
|
for _, e := range mlc.memories {
|
|
if !e.IsDeleted && e.Tier != "core" {
|
|
results = append(results, map[string]interface{}{
|
|
"id": e.ID,
|
|
})
|
|
}
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (mlc *MemLanceClient) GetSkillCandidates(minRecalls, limit int) ([]models.MemoryRecord, error) {
|
|
return nil, nil // 内存实现,暂不支持
|
|
}
|
|
|
|
func (mlc *MemLanceClient) Backup(path string) error {
|
|
return nil // 内存实现无需备份
|
|
}
|
|
|
|
func (mlc *MemLanceClient) GetAuditLog(limit int) ([]map[string]interface{}, error) {
|
|
mlc.mu.RLock()
|
|
defer mlc.mu.RUnlock()
|
|
if limit > len(mlc.auditLog) {
|
|
limit = len(mlc.auditLog)
|
|
}
|
|
return mlc.auditLog[:limit], nil
|
|
}
|
|
|
|
func (mlc *MemLanceClient) IncrementUseful(id string) {
|
|
mlc.mu.Lock()
|
|
defer mlc.mu.Unlock()
|
|
if e, ok := mlc.memories[id]; ok {
|
|
e.UsefulCount++
|
|
e.RecallCount++
|
|
}
|
|
}
|
|
|
|
func (mlc *MemLanceClient) IncrementNotUseful(id string) {
|
|
mlc.mu.Lock()
|
|
defer mlc.mu.Unlock()
|
|
if e, ok := mlc.memories[id]; ok {
|
|
e.NotUsefulCount++
|
|
}
|
|
}
|
|
|
|
func (mlc *MemLanceClient) UpdateMemoryContent(id, newContent, source string) error {
|
|
mlc.mu.Lock()
|
|
defer mlc.mu.Unlock()
|
|
if e, ok := mlc.memories[id]; ok {
|
|
e.Version++
|
|
e.VersionHistory = append(e.VersionHistory, map[string]interface{}{
|
|
"version": e.Version,
|
|
"content": newContent,
|
|
"source": source,
|
|
"reason": "corrected",
|
|
"timestamp": time.Now().Format(time.RFC3339),
|
|
})
|
|
e.Content = newContent
|
|
e.Source = source
|
|
// 重新编码
|
|
if vec, err := mlc.embed.EncodeSingle(newContent); err == nil {
|
|
e.Vector = vec
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Update 更新记录字段
|
|
func (mlc *MemLanceClient) Update(table, id string, fields map[string]any) error {
|
|
mlc.mu.Lock()
|
|
defer mlc.mu.Unlock()
|
|
if table == "memories" {
|
|
if e, ok := mlc.memories[id]; ok {
|
|
if v, ok := fields["recall_count"]; ok {
|
|
// handle $inc
|
|
if incMap, ok := v.(map[string]string); ok {
|
|
if incStr, ok := incMap["$inc"]; ok {
|
|
if incStr == "1" {
|
|
e.RecallCount++
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ─── SearchCache 兼容 ──────────────────────────────────
|
|
|
|
func (mlc *MemLanceClient) SearchCompat(queryVec []float32, topK int, namespace string) ([]interface{}, error) {
|
|
records, err := mlc.Search("memories", queryVec, topK, namespace)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
results := make([]interface{}, len(records))
|
|
for i, r := range records {
|
|
results[i] = r
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
// ─── 工具 ─────────────────────────────────────────────
|
|
|
|
func cosineSim(a, b []float32) float64 {
|
|
if len(a) != len(b) || len(a) == 0 {
|
|
return 0
|
|
}
|
|
var dot, normA, normB float64
|
|
for i := range a {
|
|
dot += float64(a[i]) * float64(b[i])
|
|
normA += float64(a[i]) * float64(a[i])
|
|
normB += float64(b[i]) * float64(b[i])
|
|
}
|
|
if normA == 0 || normB == 0 {
|
|
return 0
|
|
}
|
|
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
|
|
}
|