698 lines
21 KiB
Go
698 lines
21 KiB
Go
// 织忆 MemoryWeave — 蒸馏引擎核心
|
||
// 两阶段蒸馏：硬规则过滤 → LLM 5维度评估 → 批量蒸馏
|
||
|
||
package distill
|
||
|
||
import (
|
||
"bytes"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/xiaoxue/memoryweave/internal/metrics"
|
||
"github.com/xiaoxue/memoryweave/internal/selfoptimize"
|
||
)
|
||
|
||
// ─── 类型定义 ────────────────────────────────────────────────
|
||
|
||
// Category 记忆类别
|
||
type Category string
|
||
|
||
const (
|
||
CatSystemFact Category = "system_fact"
|
||
CatUserPref Category = "user_pref"
|
||
CatProjContext Category = "proj_context"
|
||
CatToolUsage Category = "tool_usage"
|
||
CatCodeSnippet Category = "code_snippet"
|
||
CatDecision Category = "decision"
|
||
)
|
||
|
||
// DistillInput 蒸馏输入
|
||
type DistillInput struct {
|
||
EpisodeID string
|
||
Content string
|
||
Category Category
|
||
Namespace string
|
||
AgentID string
|
||
}
|
||
|
||
// DistillResult 蒸馏输出
|
||
type DistillResult struct {
|
||
Facts []string
|
||
Entities []Entity
|
||
Score5D FiveDScore
|
||
Overall float64
|
||
Index []AAAKEntry // P4: AAAK 压缩索引（每条事实的紧凑摘要）
|
||
}
|
||
|
||
// Entity 实体
|
||
type Entity struct {
|
||
Name string `json:"name"`
|
||
Type string `json:"type"` // entity / fact / decision / skill
|
||
Properties []string `json:"properties"`
|
||
}
|
||
|
||
// FiveDScore LLM 5维评估
|
||
type FiveDScore struct {
|
||
IS float64 `json:"is"` // Information Significance
|
||
SU float64 `json:"su"` // Strategic Utility
|
||
PA float64 `json:"pa"` // Practical Applicability
|
||
VD float64 `json:"vd"` // Validation Durability
|
||
RU float64 `json:"ru"` // Recall Usability
|
||
}
|
||
|
||
// LLMResponse LLM 完整响应（5D + 实体 + 事实）
|
||
type LLMResponse struct {
|
||
IS float64 `json:"is"`
|
||
SU float64 `json:"su"`
|
||
PA float64 `json:"pa"`
|
||
VD float64 `json:"vd"`
|
||
RU float64 `json:"ru"`
|
||
Entities []string `json:"entities"`
|
||
Facts []string `json:"facts"` // 旧格式兼容
|
||
Decisions []string `json:"decisions"` // 新格式：决策结论
|
||
Conclusions []string `json:"conclusions"` // 新格式：最终结论
|
||
ActionsTaken []string `json:"actions_taken"` // 新格式：采取的行动
|
||
OpenQuestions []string `json:"open_questions"` // 新格式：悬而未决
|
||
}
|
||
|
||
// LLM 5维权重
|
||
var Weights = FiveDScore{
|
||
IS: 0.20,
|
||
SU: 0.20,
|
||
PA: 0.15,
|
||
VD: 0.25,
|
||
RU: 0.20,
|
||
}
|
||
|
||
// ─── 蒸馏引擎 ────────────────────────────────────────────────
|
||
|
||
// Engine 蒸馏引擎
|
||
type Engine struct {
|
||
mu sync.Mutex
|
||
|
||
// LLM 配置
|
||
LLMEndpoint string
|
||
LLMModel string
|
||
APIKey string
|
||
|
||
// 队列
|
||
queue []DistillInput
|
||
batchSize int
|
||
batchTimeout time.Duration
|
||
lastDistill time.Time
|
||
|
||
// P3 双缓冲：token 积累触发（LightMem 移植）
|
||
pendingTokens int
|
||
flushTokenThreshold int
|
||
|
||
// 成本控制
|
||
dailyLimit int
|
||
dailyUsed int
|
||
dailyReset time.Time
|
||
|
||
// HTTP客户端
|
||
client *http.Client
|
||
}
|
||
|
||
func NewEngine(llmEndpoint, llmModel, apiKey string) *Engine {
|
||
return &Engine{
|
||
LLMEndpoint: llmEndpoint,
|
||
LLMModel: llmModel,
|
||
APIKey: apiKey,
|
||
batchSize: 10,
|
||
batchTimeout: 5 * time.Minute,
|
||
dailyLimit: 5000,
|
||
flushTokenThreshold: 2000, // LightMem short-term buffer
|
||
client: &http.Client{Timeout: 120 * time.Second},
|
||
}
|
||
}
|
||
|
||
// estimateTokens 粗略 token 估算（中文≈1 token/字符，英文≈1 token/4字符）
|
||
// 用于 P3 双缓冲触发，不需要精确（只影响批量时机）
|
||
func estimateTokens(s string) int {
|
||
runes := len([]rune(s))
|
||
if runes == 0 {
|
||
return 0
|
||
}
|
||
// 中文按 1 token/字符，非中文按 1 token/4 字符近似
|
||
// 简单折中：rune 数 / 2
|
||
return (runes + 1) / 2
|
||
}
|
||
|
||
// Enqueue 入队
|
||
func (e *Engine) Enqueue(input DistillInput) {
|
||
e.mu.Lock()
|
||
defer e.mu.Unlock()
|
||
|
||
if !HardRulesPass(input.Content) {
|
||
return
|
||
}
|
||
|
||
e.queue = append(e.queue, input)
|
||
e.pendingTokens += estimateTokens(input.Content)
|
||
|
||
// P3 双缓冲触发：token 积累达阈值 或 条数达 batchSize 或 超时
|
||
shouldFlush := e.pendingTokens >= e.flushTokenThreshold || len(e.queue) >= e.batchSize
|
||
timeout := time.Since(e.lastDistill) > e.batchTimeout
|
||
|
||
if shouldFlush || (timeout && len(e.queue) > 0) {
|
||
go e.flush()
|
||
}
|
||
}
|
||
|
||
// flush 批量蒸馏
|
||
func (e *Engine) flush() {
|
||
e.mu.Lock()
|
||
if len(e.queue) == 0 {
|
||
e.mu.Unlock()
|
||
return
|
||
}
|
||
|
||
batch := e.queue
|
||
e.queue = nil
|
||
e.pendingTokens = 0
|
||
e.lastDistill = time.Now()
|
||
e.mu.Unlock()
|
||
|
||
log.Printf("[distill] flush START: batch=%d items, dailyUsed=%d/%d, endpoint=%s, model=%s",
|
||
len(batch), e.dailyUsed, e.dailyLimit, e.LLMEndpoint, e.LLMModel)
|
||
|
||
// Phase F: 更新 LLM 调用计数
|
||
metrics.DistillLLMCallsToday.Set(float64(e.dailyUsed))
|
||
|
||
// 成本控制检查
|
||
e.checkDailyLimit()
|
||
if e.dailyUsed >= e.dailyLimit {
|
||
log.Printf("[distill] flush FALLBACK: daily limit reached (%d/%d)", e.dailyUsed, e.dailyLimit)
|
||
results := fallbackDistill(batch)
|
||
e.emitResults(results)
|
||
return
|
||
}
|
||
|
||
// LLM 蒸馏
|
||
for _, input := range batch {
|
||
result := e.distillOne(input)
|
||
log.Printf("[distill] distillOne done: episode=%s facts=%d entities=%d",
|
||
input.EpisodeID, len(result.Facts), len(result.Entities))
|
||
e.dailyUsed++
|
||
e.emitResult(input, result)
|
||
}
|
||
log.Printf("[distill] flush DONE: processed %d items", len(batch))
|
||
}
|
||
|
||
// distillOne 蒸馏单条
|
||
func (e *Engine) distillOne(input DistillInput) DistillResult {
|
||
// 如果 LLM 端点不可用，降级
|
||
if e.LLMEndpoint == "" {
|
||
log.Printf("[distill] LLMEndpoint empty, fallback for %s", input.EpisodeID)
|
||
return fallbackSingle(input)
|
||
}
|
||
llmResp, err := e.callLLM5D(input.Content)
|
||
if err != nil {
|
||
log.Printf("[distill] callLLM5D err for %s: %v", input.EpisodeID, err)
|
||
return fallbackSingle(input)
|
||
}
|
||
|
||
score := FiveDScore{IS: llmResp.IS, SU: llmResp.SU, PA: llmResp.PA, VD: llmResp.VD, RU: llmResp.RU}
|
||
overall := score.IS*Weights.IS +
|
||
score.SU*Weights.SU +
|
||
score.PA*Weights.PA +
|
||
score.VD*Weights.VD +
|
||
score.RU*Weights.RU
|
||
|
||
log.Printf("[distill] score for %s: overall=%.3f is=%.2f su=%.2f pa=%.2f vd=%.2f ru=%.2f",
|
||
input.EpisodeID, overall, score.IS, score.SU, score.PA, score.VD, score.RU)
|
||
|
||
if overall < 0.7 && score.VD < 0.8 {
|
||
log.Printf("[distill] score below threshold for %s, skip", input.EpisodeID)
|
||
// 即使阈值未通过，仍然用 LLM 实体提取（如果有的话）
|
||
if len(llmResp.Entities) == 0 {
|
||
return DistillResult{}
|
||
}
|
||
}
|
||
|
||
// 优先使用 LLM 提取的实体和事实
|
||
var entities []Entity
|
||
var facts []string
|
||
|
||
if len(llmResp.Entities) > 0 {
|
||
for _, name := range llmResp.Entities {
|
||
entities = append(entities, Entity{
|
||
Name: name, Type: "entity", Properties: []string{"llm_extracted"},
|
||
})
|
||
}
|
||
} else {
|
||
// 降级：启发式提取
|
||
_, heuristicEntities := e.extractFacts(input.Content)
|
||
entities = heuristicEntities
|
||
}
|
||
|
||
// 事实：优先使用新的结构化字段（decisions/conclusions/actions_taken/open_questions）
|
||
// 降级：回退到 llmResp.Facts（旧格式）→ 再降级：启发式提取
|
||
hasStructuredFields := len(llmResp.Decisions) > 0 || len(llmResp.Conclusions) > 0 ||
|
||
len(llmResp.ActionsTaken) > 0 || len(llmResp.OpenQuestions) > 0
|
||
if hasStructuredFields {
|
||
// 新格式：合并 decisions/conclusions/actions_taken/open_questions 到 facts
|
||
facts = append(facts, llmResp.Decisions...)
|
||
facts = append(facts, llmResp.Conclusions...)
|
||
facts = append(facts, llmResp.ActionsTaken...)
|
||
facts = append(facts, llmResp.OpenQuestions...)
|
||
log.Printf("[distill] structured fields: decisions=%d conclusions=%d actions=%d open=%d",
|
||
len(llmResp.Decisions), len(llmResp.Conclusions),
|
||
len(llmResp.ActionsTaken), len(llmResp.OpenQuestions))
|
||
} else if len(llmResp.Facts) > 0 {
|
||
facts = llmResp.Facts
|
||
} else {
|
||
heuristicFacts, _ := e.extractFacts(input.Content)
|
||
facts = heuristicFacts
|
||
}
|
||
|
||
// P4: 生成 AAAK 压缩索引（每条事实的紧凑摘要，供召回快速定位）
|
||
index := buildAAAKIndex(facts, entities, overall)
|
||
if len(index) > 0 {
|
||
logAAAKIndex(index)
|
||
}
|
||
|
||
return DistillResult{
|
||
Facts: facts,
|
||
Entities: entities,
|
||
Score5D: score,
|
||
Overall: overall,
|
||
Index: index,
|
||
}
|
||
}
|
||
|
||
// callLLM5D 调用 LLM 进行 5维评估 + 实体/事实提取
|
||
func (e *Engine) callLLM5D(content string) (LLMResponse, error) {
|
||
// LightMem 式逐条事实提取 prompt（2026-08-11 移植）
|
||
// 精华：逐条判断含事实 → 轻量补全独立句 → 保留全部实体细节 → 推断隐含信息 → 时间区分
|
||
prompt := fmt.Sprintf(`你是一个个人信息提取器。从以下对话内容中提取所有可能的用户事实信息，以JSON格式返回。
|
||
|
||
输入格式:
|
||
[时间戳, 星期] 说话者: 消息
|
||
...
|
||
|
||
重要指令:
|
||
1. 必须按顺序逐条处理每条消息。对每条消息，判断是否包含事实信息。
|
||
- 如果包含 → 提取并改写为独立的完整句子
|
||
- 如果不包含（纯问候、填充语、无关评论）→ 跳过
|
||
- 不要因为信息看起来微小、琐碎或不重要就跳过。即使是小细节（如"用户今早喝了咖啡"）也必须保留。只有完全无意义的（如"你好"、"哈哈"、"谢谢"）才跳过。
|
||
2. 进行轻量上下文补全，使每个事实成为清晰的独立陈述：
|
||
- "user: 昨天买了苹果" → "用户昨天买了苹果。"
|
||
- "user: 我的朋友John在学医" → "用户的朋友John在学医。"
|
||
3. 保留所有具体实体和细节：
|
||
- 完整名称: "The Name of the Wind by Patrick Rothfuss"（不是"一本书"）
|
||
- 完整地点: Galway, Ireland; 北京海淀区
|
||
- 具体事件名: 慈善篮球赛、留学项目
|
||
- 数字和数量: 4年前、下个月、上周
|
||
- 公司/组织名: 某饮料公司
|
||
4. 推断隐含信息：如果多个相关条目提到 → 可以推断一般模式（保留具体事实和推断结论为独立条目）
|
||
5. 时间处理：区分提及时间（何时说的）和事件时间（何时发生的）
|
||
- 相对时间（昨天、上周、X前、下个月）→ 保留相对时间并引用消息时间戳
|
||
- 持续/永久事实 → 无需时间标注
|
||
6. 额外提取：
|
||
- decisions: 明确的决策结论（做了什么决定、选了什么方案、拒绝了什么）
|
||
- conclusions: 最终结论或答案
|
||
- actions_taken: 采取的具体行动
|
||
- open_questions: 悬而未决的问题
|
||
- entities: 提到的关键实体（系统名、工具名、人名、技术名词）
|
||
|
||
输出格式（严格JSON）:
|
||
{"facts": ["独立事实1", "独立事实2"], "decisions": ["决定1"], "conclusions": ["结论1"], "actions_taken": ["行动1"], "open_questions": ["问题1"], "entities": ["entity1", "entity2"], "is": 0.8, "su": 0.7, "pa": 0.6, "vd": 0.9, "ru": 0.7}
|
||
|
||
评分说明：is/su/pa/vd/ru 是 0.0 到 1.0 之间的浮点数（越高越好），不要用 0-10 整数。
|
||
要求：除非消息完全无意义，否则提取并输出为事实。facts 要详尽，不要只给1条摘要。
|
||
|
||
内容:
|
||
%s
|
||
`, truncate(content, 1000))
|
||
|
||
body := map[string]interface{}{
|
||
"model": e.LLMModel,
|
||
"messages": []map[string]string{
|
||
{"role": "user", "content": prompt},
|
||
},
|
||
"temperature": 0.2,
|
||
"max_tokens": 800,
|
||
}
|
||
|
||
jsonBody, err := json.Marshal(body)
|
||
if err != nil {
|
||
return LLMResponse{}, err
|
||
}
|
||
|
||
req, err := http.NewRequest("POST", e.LLMEndpoint, bytes.NewReader(jsonBody))
|
||
if err != nil {
|
||
return LLMResponse{}, err
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
if e.APIKey != "" {
|
||
req.Header.Set("Authorization", "Bearer "+e.APIKey)
|
||
}
|
||
|
||
resp, err := e.client.Do(req)
|
||
if err != nil {
|
||
return LLMResponse{}, err
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
respBody, _ := io.ReadAll(resp.Body)
|
||
|
||
var result struct {
|
||
Choices []struct {
|
||
Message struct {
|
||
Content string `json:"content"`
|
||
ReasoningContent string `json:"reasoning_content"`
|
||
// OpenAI 系 reasoning 模型（gpt-oss 等）用 "reasoning" 字段，不是 "reasoning_content"
|
||
Reasoning string `json:"reasoning"`
|
||
} `json:"message"`
|
||
} `json:"choices"`
|
||
}
|
||
|
||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||
return LLMResponse{}, err
|
||
}
|
||
|
||
if len(result.Choices) == 0 {
|
||
return LLMResponse{}, fmt.Errorf("no choices in LLM response")
|
||
}
|
||
|
||
var llmResp LLMResponse
|
||
llmContent := result.Choices[0].Message.Content
|
||
if llmContent == "" {
|
||
llmContent = result.Choices[0].Message.ReasoningContent
|
||
}
|
||
if llmContent == "" {
|
||
llmContent = result.Choices[0].Message.Reasoning
|
||
}
|
||
// 剥离 markdown code fence（minimax 等模型习惯用 ```json 包裹）
|
||
llmContent = strings.TrimSpace(llmContent)
|
||
llmContent = strings.TrimPrefix(llmContent, "```json")
|
||
llmContent = strings.TrimPrefix(llmContent, "```")
|
||
llmContent = strings.TrimSuffix(llmContent, "```")
|
||
llmContent = strings.TrimSpace(llmContent)
|
||
// 健壮剥离：找第一个 { 和最后一个 } 截取（模型可能在 JSON 后加 markdown/注释）
|
||
if idx := strings.Index(llmContent, "{"); idx > 0 {
|
||
llmContent = llmContent[idx:]
|
||
}
|
||
if idx := strings.LastIndex(llmContent, "}"); idx >= 0 && idx < len(llmContent)-1 {
|
||
llmContent = llmContent[:idx+1]
|
||
}
|
||
llmContent = strings.TrimSpace(llmContent)
|
||
if err := json.Unmarshal([]byte(llmContent), &llmResp); err != nil {
|
||
log.Printf("[distill] LLM JSON parse error: %v | content=%q", err, truncate(llmContent, 200))
|
||
return LLMResponse{}, fmt.Errorf("parse score: %w", err)
|
||
}
|
||
|
||
// 记录实体和事实数量
|
||
if len(llmResp.Entities) > 0 {
|
||
log.Printf("[distill] LLM entities for content: %d entities", len(llmResp.Entities))
|
||
}
|
||
if len(llmResp.Facts) > 0 {
|
||
log.Printf("[distill] LLM facts for content: %d facts", len(llmResp.Facts))
|
||
}
|
||
|
||
return llmResp, nil
|
||
}
|
||
|
||
// extractFacts 从内容中提取事实和实体（关键词 + 命名实体启发式）
|
||
func (e *Engine) extractFacts(content string) ([]string, []Entity) {
|
||
var facts []string
|
||
var entities []Entity
|
||
|
||
if len(content) > 20 {
|
||
facts = append(facts, truncate(content, 200))
|
||
}
|
||
|
||
// 启发式实体提取：提取大写单词、数字组合、中文命名实体
|
||
words := strings.Fields(content)
|
||
seen := make(map[string]bool)
|
||
for _, w := range words {
|
||
w = strings.Trim(w, ",.;:!?，。；：！？、\"'()（）[]【】")
|
||
if len(w) < 2 {
|
||
continue
|
||
}
|
||
// 大写字母开头（英文命名实体：Hermes, ComfyUI, Redis 等）
|
||
runes := []rune(w)
|
||
if len(runes) >= 2 && runes[0] >= 'A' && runes[0] <= 'Z' {
|
||
normalized := strings.ToLower(w)
|
||
if !seen[normalized] && !isStopWord(normalized) {
|
||
seen[normalized] = true
|
||
entities = append(entities, Entity{
|
||
Name: w, Type: "entity", Properties: []string{"extracted"},
|
||
})
|
||
}
|
||
}
|
||
// 中文实体（2-20 个纯中文字符，不含标点）
|
||
cleanChinese := stripNonChinese(w)
|
||
if len(cleanChinese) >= 2 && len(cleanChinese) <= 20 {
|
||
if !seen[cleanChinese] {
|
||
seen[cleanChinese] = true
|
||
isTech := containsAny(cleanChinese, []string{"端口", "配置", "系统", "内存", "显卡", "服务", "记忆", "织忆", "飞书", "版本", "模型", "工具", "项目", "安装", "部署", "目录", "路径", "开发", "语言"})
|
||
etype := "entity"
|
||
if isTech { etype = "fact" }
|
||
entities = append(entities, Entity{
|
||
Name: cleanChinese, Type: etype, Properties: []string{"extracted"},
|
||
})
|
||
}
|
||
}
|
||
// 数字/字母组合或纯数字（如 3050, 4GB, 8188, 7821）
|
||
if isTechToken(w) || isAllDigits(w) {
|
||
if !seen[w] {
|
||
seen[w] = true
|
||
entities = append(entities, Entity{
|
||
Name: w, Type: "entity", Properties: []string{"technical"},
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
return facts, entities
|
||
}
|
||
|
||
// isChineseSequence 判断是否为连续的中文字符串（长度在 minLen 到 maxLen 之间）
|
||
func isChineseSequence(s string, minLen, maxLen int) bool {
|
||
runes := []rune(s)
|
||
if len(runes) < minLen || len(runes) > maxLen {
|
||
return false
|
||
}
|
||
for _, r := range runes {
|
||
if r < 0x4E00 || r > 0x9FFF {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// isTechToken 判断是否为技术标记（包含数字+字母组合）
|
||
func isTechToken(s string) bool {
|
||
hasDigit := false
|
||
hasLetter := false
|
||
for _, r := range s {
|
||
if r >= '0' && r <= '9' { hasDigit = true }
|
||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') { hasLetter = true }
|
||
}
|
||
return hasDigit && hasLetter
|
||
}
|
||
|
||
// containsAny 判断字符串是否包含任意一个子串
|
||
func containsAny(s string, subs []string) bool {
|
||
for _, sub := range subs {
|
||
if strings.Contains(s, sub) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// isStopWord 判断是否为停用词
|
||
func isStopWord(w string) bool {
|
||
stops := map[string]bool{
|
||
"the": true, "a": true, "an": true, "is": true, "are": true,
|
||
"was": true, "were": true, "be": true, "been": true, "being": true,
|
||
"have": true, "has": true, "had": true, "do": true, "does": true,
|
||
"did": true, "will": true, "would": true, "could": true, "should": true,
|
||
"may": true, "might": true, "can": true, "shall": true,
|
||
"this": true, "that": true, "these": true, "those": true,
|
||
"it": true, "its": true, "they": true, "them": true, "their": true,
|
||
"we": true, "our": true, "you": true, "your": true, "he": true, "she": true,
|
||
"his": true, "her": true, "and": true, "or": true, "but": true,
|
||
"not": true, "no": true, "if": true, "then": true, "else": true,
|
||
"for": true, "with": true, "as": true, "at": true, "by": true,
|
||
"from": true, "in": true, "of": true, "on": true, "to": true, "up": true,
|
||
}
|
||
return stops[w]
|
||
}
|
||
|
||
// stripNonChinese 只保留中文字符
|
||
func stripNonChinese(s string) string {
|
||
var buf []rune
|
||
for _, r := range s {
|
||
if r >= 0x4E00 && r <= 0x9FFF {
|
||
buf = append(buf, r)
|
||
}
|
||
}
|
||
return string(buf)
|
||
}
|
||
|
||
// isAllDigits 判断是否为纯数字
|
||
func isAllDigits(s string) bool {
|
||
if len(s) == 0 {
|
||
return false
|
||
}
|
||
for _, r := range s {
|
||
if r < '0' || r > '9' {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// checkDailyLimit 每日限额检查
|
||
func (e *Engine) checkDailyLimit() {
|
||
now := time.Now()
|
||
if now.Sub(e.dailyReset) > 24*time.Hour {
|
||
e.dailyUsed = 0
|
||
e.dailyReset = now
|
||
}
|
||
}
|
||
|
||
// emitResult 发送蒸馏结果（回调）
|
||
func (e *Engine) emitResult(input DistillInput, result DistillResult) {
|
||
if len(result.Facts) == 0 {
|
||
return
|
||
}
|
||
// 由注册的回调处理
|
||
if OnDistillComplete != nil {
|
||
OnDistillComplete(input, result)
|
||
}
|
||
|
||
// Phase F: 蒸馏完成后更新队列深度和冲突计数
|
||
metrics.DistillQueueDepth.Set(float64(e.QueueLen()))
|
||
|
||
// Phase F: 对低分记忆触发质量监控检查 (score < 0.7 的记忆视为潜在低质量)
|
||
if result.Overall > 0 && result.Overall < 0.7 {
|
||
feedbackCount := selfoptimize.Dash.UsefulCount + selfoptimize.Dash.NotUsefulCount
|
||
if record := selfoptimize.QualityMonitor.Check(input.EpisodeID, result.Overall, feedbackCount); record != nil {
|
||
log.Printf("[quality] distill low-score flagged: episode=%s score=%.2f status=%s",
|
||
input.EpisodeID, result.Overall, record.Status)
|
||
}
|
||
}
|
||
}
|
||
|
||
// emitResults 批量发送
|
||
func (e *Engine) emitResults(results map[DistillInput]DistillResult) {
|
||
for input, result := range results {
|
||
e.emitResult(input, result)
|
||
}
|
||
}
|
||
|
||
// OnDistillComplete 全局蒸馏完成回调
|
||
var OnDistillComplete func(DistillInput, DistillResult)
|
||
|
||
// ─── 辅助 ────────────────────────────────────────────────
|
||
|
||
func truncate(s string, maxLen int) string {
|
||
runes := []rune(s)
|
||
if len(runes) <= maxLen {
|
||
return s
|
||
}
|
||
return string(runes[:maxLen]) + "..."
|
||
}
|
||
|
||
// fallbackSingle 降级蒸馏（无 LLM）
|
||
func fallbackSingle(input DistillInput) DistillResult {
|
||
var facts []string
|
||
if len(input.Content) > 20 {
|
||
facts = append(facts, truncate(input.Content, 100))
|
||
}
|
||
return DistillResult{Facts: facts}
|
||
}
|
||
|
||
// fallbackDistill 批量降级蒸馏
|
||
func fallbackDistill(inputs []DistillInput) map[DistillInput]DistillResult {
|
||
results := make(map[DistillInput]DistillResult)
|
||
for _, input := range inputs {
|
||
results[input] = fallbackSingle(input)
|
||
}
|
||
return results
|
||
}
|
||
|
||
// ─── 端点导出方法（G8-G9 配套）───────────────────────────────
|
||
|
||
// QueueLen 返回当前队列长度
|
||
func (e *Engine) QueueLen() int {
|
||
e.mu.Lock()
|
||
defer e.mu.Unlock()
|
||
return len(e.queue)
|
||
}
|
||
|
||
// QueueItems 返回队列内容（摘要）
|
||
func (e *Engine) QueueItems() []map[string]string {
|
||
e.mu.Lock()
|
||
defer e.mu.Unlock()
|
||
items := make([]map[string]string, len(e.queue))
|
||
for i, q := range e.queue {
|
||
content := q.Content
|
||
if len(content) > 80 {
|
||
content = content[:80] + "..."
|
||
}
|
||
items[i] = map[string]string{
|
||
"episode_id": q.EpisodeID,
|
||
"content": content,
|
||
"category": string(q.Category),
|
||
}
|
||
}
|
||
return items
|
||
}
|
||
|
||
// GetStatus 返回引擎运行时状态
|
||
func (e *Engine) GetStatus() map[string]interface{} {
|
||
e.mu.Lock()
|
||
defer e.mu.Unlock()
|
||
return map[string]interface{}{
|
||
"queue_len": len(e.queue),
|
||
"batch_size": e.batchSize,
|
||
"last_distill": e.lastDistill.Format(time.RFC3339),
|
||
"daily_used": e.dailyUsed,
|
||
"daily_limit": e.dailyLimit,
|
||
"daily_remaining": e.dailyLimit - e.dailyUsed,
|
||
}
|
||
}
|
||
|
||
// GetQuota 返回配额（基于 Engine 自身追踪）
|
||
func (e *Engine) GetQuota() map[string]interface{} {
|
||
e.mu.Lock()
|
||
used := e.dailyUsed
|
||
limit := e.dailyLimit
|
||
e.mu.Unlock()
|
||
remain := limit - used
|
||
if remain < 0 {
|
||
remain = 0
|
||
}
|
||
pct := float64(used) / float64(limit) * 100
|
||
if limit == 0 {
|
||
pct = 0
|
||
}
|
||
status := "normal"
|
||
if used >= limit {
|
||
status = "exceeded"
|
||
} else if pct >= 80 {
|
||
status = "near"
|
||
}
|
||
return map[string]interface{}{
|
||
"remaining": remain,
|
||
"used": used,
|
||
"limit": limit,
|
||
"percent": pct,
|
||
"near_limit": pct >= 80,
|
||
"status": status,
|
||
}
|
||
}
|