feat(distill): P4 AAAK压缩索引 + P5 Markdown导出层
P4: 借鉴 mempalace AAAK dialect — 每条事实生成紧凑索引(实体|关键词|权重|类型) P5: 借鉴 EverOS md真相层 — /api/v1/memories/export 导出人可读 Markdown
This commit is contained in:
小唯
parent
82c3d25423
commit
38701748d1
|
|
@ -460,4 +460,56 @@ func (aa *AdminAPI) ListMemories(w http.ResponseWriter, r *http.Request) {
|
|||
respond(w, 200, map[string]interface{}{
|
||||
"memories": items, "count": len(items), "limit": limit,
|
||||
})
|
||||
}
|
||||
|
||||
// ExportMemoriesMD 导出记忆为 Markdown（P5：EverOS 式 md 真相层）
|
||||
// GET /api/v1/memories/export?namespace=hermes-main&limit=1000&format=md
|
||||
// 输出人可读的 Markdown 文档：记忆可迁移、可备份、可人工审查
|
||||
func (aa *AdminAPI) ExportMemoriesMD(w http.ResponseWriter, r *http.Request) {
|
||||
ns := r.URL.Query().Get("namespace")
|
||||
limit := 1000
|
||||
if l := r.URL.Query().Get("limit"); l != "" {
|
||||
if v, err := fmt.Sscanf(l, "%d", &limit); err != nil || v != 1 || limit < 1 {
|
||||
limit = 1000
|
||||
}
|
||||
if limit > 5000 {
|
||||
limit = 5000
|
||||
}
|
||||
}
|
||||
zeroVec := make([]float32, 1024)
|
||||
results, err := aa.LanceDB.Search("memories", zeroVec, limit, ns)
|
||||
if err != nil {
|
||||
respondError(w, 500, "export memories: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("# 织忆记忆导出\n\n")
|
||||
sb.WriteString(fmt.Sprintf("> 导出时间: %s\n", time.Now().Format("2006-01-02 15:04:05")))
|
||||
sb.WriteString(fmt.Sprintf("> 命名空间: %s | 条数: %d\n\n---\n\n", orDefault(ns, "all"), len(results)))
|
||||
|
||||
for i, m := range results {
|
||||
sb.WriteString(fmt.Sprintf("## M%d — %s\n\n", i+1, m.ID))
|
||||
sb.WriteString(fmt.Sprintf("- **分类**: %s\n", orDefault(m.Category, "unknown")))
|
||||
if m.Namespace != "" {
|
||||
sb.WriteString(fmt.Sprintf("- **命名空间**: %s\n", m.Namespace))
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("- **重要性**: %.2f\n", m.Importance))
|
||||
sb.WriteString(fmt.Sprintf("- **时间**: %s\n", m.CreatedAt.Format("2006-01-02 15:04:05")))
|
||||
sb.WriteString("\n### 内容\n\n")
|
||||
sb.WriteString(m.Content)
|
||||
sb.WriteString("\n\n---\n\n")
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
|
||||
w.Header().Set("Content-Disposition", "attachment; filename=zhiyi-memories-"+time.Now().Format("20060102")+".md")
|
||||
w.WriteHeader(200)
|
||||
w.Write([]byte(sb.String()))
|
||||
}
|
||||
|
||||
func orDefault(s, def string) string {
|
||||
if s == "" {
|
||||
return def
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
|
@ -913,6 +913,7 @@ func NewServer() http.Handler {
|
|||
})
|
||||
// GET /api/v1/memories — list all memories (zero-vector search, for plugin compat)
|
||||
mux.HandleFunc("/api/v1/memories", adminAPI.ListMemories)
|
||||
mux.HandleFunc("/api/v1/memories/export", adminAPI.ExportMemoriesMD)
|
||||
// /api/v1/memory/{id}/versions — existing version history endpoint
|
||||
mux.HandleFunc("/api/v1/memory/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == "GET" {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,157 @@
|
|||
package distill
|
||||
|
||||
import (
|
||||
"log"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// AAAKEntry AAAK 风格压缩索引条目（借鉴 mempalace/dialect.py 的 AAAK 设计）
|
||||
// 目标：为每条事实生成紧凑结构化摘要，LLM 可读、无需解码器，
|
||||
// 召回时先扫索引定位相关事实，再读原文（索引层指向内容层）。
|
||||
type AAAKEntry struct {
|
||||
FactID string `json:"fact_id"` // 事实编号（F1, F2, ...）
|
||||
Primary string `json:"primary"` // 主实体（最重要实体，如人名/项目名）
|
||||
Entities []string `json:"entities"` // 全部相关实体
|
||||
Keywords []string `json:"keywords"` // 主题关键词（2-4 个）
|
||||
Quote string `json:"quote"` // 关键短语（截断 ≤40 字）
|
||||
Weight float64 `json:"weight"` // 权重（由 5D 分数综合）
|
||||
Kind string `json:"kind"` // 类型：fact / decision / action / question / conclusion
|
||||
}
|
||||
|
||||
// buildAAAKIndex 为事实列表生成 AAAK 压缩索引
|
||||
func buildAAAKIndex(facts []string, entities []Entity, overall float64) []AAAKEntry {
|
||||
entries := make([]AAAKEntry, 0, len(facts))
|
||||
entityNames := make([]string, 0, len(entities))
|
||||
for _, e := range entities {
|
||||
if e.Name != "" {
|
||||
entityNames = append(entityNames, e.Name)
|
||||
}
|
||||
}
|
||||
|
||||
for i, fact := range facts {
|
||||
if strings.TrimSpace(fact) == "" {
|
||||
continue
|
||||
}
|
||||
entry := AAAKEntry{
|
||||
FactID: "F" + itoa(i+1),
|
||||
Primary: pickPrimary(fact, entityNames),
|
||||
Entities: pickRelatedEntities(fact, entityNames, 4),
|
||||
Keywords: pickKeywords(fact, 3),
|
||||
Quote: truncate(fact, 40),
|
||||
Weight: overall,
|
||||
Kind: classifyFact(fact),
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// classifyFact 事实类型分类
|
||||
func classifyFact(fact string) string {
|
||||
switch {
|
||||
case containsAny(fact, []string{"决定", "选择", "采用", "确定", "配置", "改", "换"}):
|
||||
return "decision"
|
||||
case containsAny(fact, []string{"完成", "实现", "部署", "安装", "修复", "上线", "验证", "测试"}):
|
||||
return "action"
|
||||
case containsAny(fact, []string{"?", "？", "是否", "吗", "未", "待", "需要"}):
|
||||
return "question"
|
||||
case containsAny(fact, []string{"结论", "因此", "所以", "总之", "意味着"}):
|
||||
return "conclusion"
|
||||
default:
|
||||
return "fact"
|
||||
}
|
||||
}
|
||||
|
||||
// pickPrimary 选取主实体（事实中第一个出现的已知实体，否则第一个词）
|
||||
func pickPrimary(fact string, entityNames []string) string {
|
||||
for _, name := range entityNames {
|
||||
if name != "" && strings.Contains(fact, name) {
|
||||
return name
|
||||
}
|
||||
}
|
||||
// 退而取第一个非停用词 token
|
||||
fields := strings.Fields(fact)
|
||||
for _, f := range fields {
|
||||
clean := strings.Trim(f, ",.;:!?，。；：！？、\"'()（）[]【】")
|
||||
if len([]rune(clean)) >= 2 && !isStopWord(strings.ToLower(clean)) {
|
||||
return clean
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// pickRelatedEntities 选取事实中出现的相关实体（最多 max 个）
|
||||
func pickRelatedEntities(fact string, entityNames []string, max int) []string {
|
||||
var picked []string
|
||||
for _, name := range entityNames {
|
||||
if name != "" && strings.Contains(fact, name) {
|
||||
picked = append(picked, name)
|
||||
if len(picked) >= max {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return picked
|
||||
}
|
||||
|
||||
// pickKeywords 提取主题关键词（从事实中挑有意义的词，最多 max 个）
|
||||
func pickKeywords(fact string, max int) []string {
|
||||
var kws []string
|
||||
seen := make(map[string]bool)
|
||||
fields := strings.Fields(fact)
|
||||
for _, f := range fields {
|
||||
clean := strings.Trim(f, ",.;:!?，。；：！？、\"'()（）[]【】")
|
||||
runes := []rune(clean)
|
||||
if len(runes) < 2 || len(runes) > 10 {
|
||||
continue
|
||||
}
|
||||
// 跳过纯标点/停用词
|
||||
if isStopWord(strings.ToLower(clean)) || isPunctuationOnly(clean) {
|
||||
continue
|
||||
}
|
||||
// 优先中文实体和技术词
|
||||
key := strings.ToLower(clean)
|
||||
if !seen[key] {
|
||||
seen[key] = true
|
||||
kws = append(kws, clean)
|
||||
if len(kws) >= max {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return kws
|
||||
}
|
||||
|
||||
func isPunctuationOnly(s string) bool {
|
||||
for _, r := range s {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// itoa 简单整数转字符串（避免引入 strconv 依赖之外的复杂度）
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
digits := []byte{}
|
||||
for n > 0 {
|
||||
digits = append([]byte{byte('0' + n%10)}, digits...)
|
||||
n /= 10
|
||||
}
|
||||
return string(digits)
|
||||
}
|
||||
|
||||
// logAAAKIndex 输出索引（调试用）
|
||||
func logAAAKIndex(entries []AAAKEntry) {
|
||||
if len(entries) == 0 {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
log.Printf("[aaak] %s|%s|%s|%.2f|%s",
|
||||
e.FactID, e.Primary, strings.Join(e.Keywords, ","), e.Weight, e.Kind)
|
||||
}
|
||||
}
|
||||
|
|
@ -47,6 +47,7 @@ type DistillResult struct {
|
|||
Entities []Entity
|
||||
Score5D FiveDScore
|
||||
Overall float64
|
||||
Index []AAAKEntry // P4: AAAK 压缩索引（每条事实的紧凑摘要）
|
||||
}
|
||||
|
||||
// Entity 实体
|
||||
|
|
@ -272,11 +273,18 @@ func (e *Engine) distillOne(input DistillInput) DistillResult {
|
|||
facts = heuristicFacts
|
||||
}
|
||||
|
||||
// P4: 生成 AAAK 压缩索引（每条事实的紧凑摘要，供召回快速定位）
|
||||
index := buildAAAKIndex(facts, entities, overall)
|
||||
if len(index) > 0 {
|
||||
logAAAKIndex(index)
|
||||
}
|
||||
|
||||
return DistillResult{
|
||||
Facts: facts,
|
||||
Entities: entities,
|
||||
Score5D: score,
|
||||
Overall: overall,
|
||||
Index: index,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue