fix(v3.0): Distilled双态移除+冲突自动触发+版本3.0+冗余消除
- Distilled模型: 删除status/confidence/created_at/updated_at字段 - commit时自动触发ConflictDetector.scan_conflicts(无需手动调用) - version: 2.6 → 3.0 (server.py + admin.py) - 冗余消除: commit时overlap>0.8自动合并(标记deprecated)
This commit is contained in:
parent
20126277c0
commit
f31cf32515
File diff suppressed because one or more lines are too long
|
|
@ -480,6 +480,7 @@
|
|||
{"id": "c4aa2663-4361-40ca-be3e-da9e1ab54954", "text": "验证新索引包含 tier 和 agent_id 字段 验证新索引包含 tier 和 agent_id 字段", "facts": ["验证新索引包含 tier 和 agent_id 字段"], "category": "distilled", "timestamp": "2026-05-26T13:36:37.033096", "tier": "normal", "agent_id": "hermes"}
|
||||
{"id": "aee6ad1e-f6cb-45a4-9447-b3eecd36746e", "text": "验证核心记忆保护:这是牧尘最重要的身份配置,tier=core永不衰减 验证核心记忆保护:这是牧尘最重要的身份配置,tier=core永不衰减", "facts": ["验证核心记忆保护:这是牧尘最重要的身份配置,tier=core永不衰减"], "category": "distilled", "timestamp": "2026-05-26T13:37:01.271110", "tier": "normal", "agent_id": "hermes"}
|
||||
{"id": "1b68146b-a410-4e21-9911-4869dd8fd479", "text": "这是最新的核心记忆测试:tier字段是否正确传递到distill 这是最新的核心记忆测试:tier字段是否正确传递到distill", "facts": ["这是最新的核心记忆测试:tier字段是否正确传递到distill"], "category": "distilled", "timestamp": "2026-05-26T13:48:12.903755", "tier": "core", "agent_id": "hermes"}
|
||||
{"id": "32a24861-bdaf-486d-b8d0-1b0921543136", "text": "测试冲突检测:这是一条关于Tailscale的冲突记忆 Tailscale是网络工具", "facts": ["Tailscale是网络工具"], "category": "episodes", "timestamp": "2026-05-26T14:44:53.225906", "tier": "normal", "agent_id": "hermes"}
|
||||
{"id": "6a6ea95c-7426-4caf-afdd-920f22ed6994", "text": "test memory ", "facts": [], "category": "episodes", "timestamp": "2026-05-25T02:24:54.078945", "tier": "normal", "agent_id": "hermes"}
|
||||
{"id": "e6d172bd-01f6-4eee-8b59-bb3aa83fb34a", "text": "牧尘测试记忆2026 ", "facts": [], "category": "episodes", "timestamp": "2026-05-25T09:14:35.517132", "tier": "normal", "agent_id": "hermes"}
|
||||
{"id": "4980e5cc-47de-4317-be4c-99291a2112b5", "text": "牧尘测试记忆2026-05-25 ", "facts": [], "category": "episodes", "timestamp": "2026-05-25T09:15:17.314559", "tier": "normal", "agent_id": "hermes"}
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -1 +1 @@
|
|||
{"model_name": "bge-m3", "doc_count": 491, "vector_dim": 1024, "index_type": "FlatIP", "updated_at": "2026-05-26T13:48:13.689259"}
|
||||
{"model_name": "bge-m3", "doc_count": 492, "vector_dim": 1024, "index_type": "FlatIP", "updated_at": "2026-05-26T14:44:54.107976"}
|
||||
|
|
@ -25,5 +25,5 @@ def get_config():
|
|||
'decay_rate': 0.015,
|
||||
'decay_floor': 0.1,
|
||||
'distill_threshold': 0.7,
|
||||
'version': '2.6'
|
||||
'version': '3.0'
|
||||
}
|
||||
|
|
@ -128,9 +128,33 @@ def commit_memory(req: CommitRequest):
|
|||
distilled_id = None
|
||||
try:
|
||||
distilled = _engine.distill_episode(episode)
|
||||
_store.append('distilled', distilled.to_dict())
|
||||
distilled_id = distilled.id
|
||||
|
||||
# 冗余消除:自动合并相似度 > 0.8 的记忆(保留最新)
|
||||
try:
|
||||
new_summary = distilled.summary
|
||||
new_content_prefix = (distilled.summary or distilled.to_dict().get('content', '')[:80])
|
||||
for record in _store.search_recent('distilled', '', months_back=1, limit=200):
|
||||
if record.get('id') == distilled.id:
|
||||
continue
|
||||
old_prefix = (record.get('summary') or record.get('content', ''))[:80]
|
||||
if not old_prefix or not new_content_prefix:
|
||||
continue
|
||||
# overlap coefficient
|
||||
old_chars = set(old_prefix)
|
||||
new_chars = set(new_content_prefix)
|
||||
intersection = len(old_chars & new_chars)
|
||||
overlap = intersection / min(len(old_chars), len(new_chars)) if min(len(old_chars), len(new_chars)) > 0 else 0
|
||||
if overlap > 0.8:
|
||||
from storage.tombstone import TombstoneStore
|
||||
TombstoneStore().mark_deleted(record.get('id', ''), reason=f'冗余消除: 与 {distilled.id[:16]} 相似度 {overlap:.2f}')
|
||||
print(f"[Dedupe] 冗余合并: {record.get('id', '')[:20]} → {distilled.id[:20]} (overlap={overlap:.2f})")
|
||||
break # 每条新记忆最多合并一条旧记录
|
||||
except Exception as e:
|
||||
print(f"[Dedupe] 冗余消除失败: {e}")
|
||||
|
||||
_store.append('distilled', distilled.to_dict())
|
||||
|
||||
# Redis 存储 distilled(多实例共享)
|
||||
if redis_store:
|
||||
try:
|
||||
|
|
@ -140,6 +164,23 @@ def commit_memory(req: CommitRequest):
|
|||
except Exception as e:
|
||||
print(f"[Redis] distilled 存储/发布失败: {e}")
|
||||
|
||||
# 自动冲突检测(commit 后立即执行,无需手动调用 /conflicts/scan)
|
||||
try:
|
||||
from distill.conflicts import ConflictDetector
|
||||
detector = ConflictDetector()
|
||||
all_distilled = list(_store.search_recent('distilled', '', months_back=1, limit=1000))
|
||||
class FakeDistilled:
|
||||
def __init__(self, d):
|
||||
self.id = d.get('id', '')
|
||||
self.entities = d.get('entities', [])
|
||||
self.facts = d.get('facts', [])
|
||||
conflicts = detector.scan_conflicts(FakeDistilled(distilled.to_dict()), all_distilled)
|
||||
if conflicts:
|
||||
for c in conflicts:
|
||||
print(f"[Conflict] 检测到冲突: {c.id} type={c.type} entity={c.entity}")
|
||||
except Exception as e:
|
||||
print(f"[Conflict] 冲突检测失败: {e}")
|
||||
|
||||
# 全量索引重建(每次 commit 都重建,保证 FAISS 索引和数据完全一致)
|
||||
# 从 JSONLShardStore 读取完整 episodes + distilled(最新状态)
|
||||
from storage.bge_embedder import get_bge_client
|
||||
|
|
|
|||
|
|
@ -68,11 +68,11 @@ app.include_router(graph.router, prefix='/api/v1', tags=['graph'])
|
|||
|
||||
@app.get('/health')
|
||||
def health():
|
||||
return {'status': 'ok', 'service': '织忆 MemoryWeave', 'version': '2.6'}
|
||||
return {'status': 'ok', 'service': '织忆 MemoryWeave', 'version': '3.0'}
|
||||
|
||||
@app.get('/')
|
||||
def root():
|
||||
return {'service': '织忆 MemoryWeave', 'version': '2.6', 'docs': '/docs'}
|
||||
return {'service': '织忆 MemoryWeave', 'version': '3.0', 'docs': '/docs'}
|
||||
|
||||
def main():
|
||||
import uvicorn
|
||||
|
|
|
|||
|
|
@ -116,11 +116,7 @@ class DistillEngine:
|
|||
summary=episode.content[:200],
|
||||
entities=entities,
|
||||
facts=facts,
|
||||
confidence=eval_result['overall'],
|
||||
status='distilled',
|
||||
importance=1 if eval_result['should_distill'] else 0,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
tier=episode.tier,
|
||||
agent_id=episode.metadata.get('agent_id', 'hermes'),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,9 +6,6 @@ TYPE_REQUEST = 'request'
|
|||
TYPE_FACT = 'fact'
|
||||
TYPE_PATTERN = 'pattern'
|
||||
|
||||
STATUS_PENDING = 'pending'
|
||||
STATUS_VALIDATED = 'validated'
|
||||
STATUS_DEPRECATED = 'deprecated'
|
||||
|
||||
@dataclass
|
||||
class Distilled:
|
||||
|
|
@ -18,13 +15,10 @@ class Distilled:
|
|||
summary: str
|
||||
entities: list[str] = field(default_factory=list)
|
||||
facts: list[str] = field(default_factory=list)
|
||||
confidence: float = 0.5
|
||||
status: str = 'pending'
|
||||
importance: int = 0
|
||||
created_at: datetime = field(default_factory=datetime.now)
|
||||
updated_at: datetime = field(default_factory=datetime.now)
|
||||
tier: str = 'normal' # 继承自 episode,core=永不衰减
|
||||
agent_id: str = 'hermes' # 继承自 episode,用于隐私隔离
|
||||
importance: float = 0.0
|
||||
tier: str = 'normal'
|
||||
agent_id: str = 'hermes'
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
|
|
@ -34,20 +28,18 @@ class Distilled:
|
|||
'summary': self.summary,
|
||||
'entities': self.entities,
|
||||
'facts': self.facts,
|
||||
'confidence': self.confidence,
|
||||
'status': self.status,
|
||||
'importance': self.importance,
|
||||
'created_at': self.created_at.isoformat(),
|
||||
'updated_at': self.updated_at.isoformat(),
|
||||
'tier': self.tier,
|
||||
'agent_id': self.agent_id,
|
||||
'metadata': self.metadata,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict):
|
||||
d = d.copy()
|
||||
d['created_at'] = datetime.fromisoformat(d['created_at'])
|
||||
d['updated_at'] = datetime.fromisoformat(d['updated_at'])
|
||||
# 兼容旧记录(可能含已删除字段)
|
||||
for old_field in ('confidence', 'status', 'created_at', 'updated_at'):
|
||||
d.pop(old_field, None)
|
||||
if 'tier' not in d:
|
||||
d['tier'] = 'normal'
|
||||
if 'agent_id' not in d:
|
||||
|
|
|
|||
Loading…
Reference in New Issue