fix(v3.0): Distilled双态移除+冲突自动触发+版本3.0+冗余消除

- Distilled模型: 删除status/confidence/created_at/updated_at字段
- commit时自动触发ConflictDetector.scan_conflicts(无需手动调用)
- version: 2.6 → 3.0 (server.py + admin.py)
- 冗余消除: commit时overlap>0.8自动合并(标记deprecated)
This commit is contained in:
小唯 A06 2026-05-26 14:45:26 +08:00
parent 20126277c0
commit f31cf32515
9 changed files with 57 additions and 25 deletions

File diff suppressed because one or more lines are too long

View File

@ -480,6 +480,7 @@
{"id": "c4aa2663-4361-40ca-be3e-da9e1ab54954", "text": "验证新索引包含 tier 和 agent_id 字段 验证新索引包含 tier 和 agent_id 字段", "facts": ["验证新索引包含 tier 和 agent_id 字段"], "category": "distilled", "timestamp": "2026-05-26T13:36:37.033096", "tier": "normal", "agent_id": "hermes"}
{"id": "aee6ad1e-f6cb-45a4-9447-b3eecd36746e", "text": "验证核心记忆保护这是牧尘最重要的身份配置tier=core永不衰减 验证核心记忆保护这是牧尘最重要的身份配置tier=core永不衰减", "facts": ["验证核心记忆保护这是牧尘最重要的身份配置tier=core永不衰减"], "category": "distilled", "timestamp": "2026-05-26T13:37:01.271110", "tier": "normal", "agent_id": "hermes"}
{"id": "1b68146b-a410-4e21-9911-4869dd8fd479", "text": "这是最新的核心记忆测试tier字段是否正确传递到distill 这是最新的核心记忆测试tier字段是否正确传递到distill", "facts": ["这是最新的核心记忆测试tier字段是否正确传递到distill"], "category": "distilled", "timestamp": "2026-05-26T13:48:12.903755", "tier": "core", "agent_id": "hermes"}
{"id": "32a24861-bdaf-486d-b8d0-1b0921543136", "text": "测试冲突检测这是一条关于Tailscale的冲突记忆 Tailscale是网络工具", "facts": ["Tailscale是网络工具"], "category": "episodes", "timestamp": "2026-05-26T14:44:53.225906", "tier": "normal", "agent_id": "hermes"}
{"id": "6a6ea95c-7426-4caf-afdd-920f22ed6994", "text": "test memory ", "facts": [], "category": "episodes", "timestamp": "2026-05-25T02:24:54.078945", "tier": "normal", "agent_id": "hermes"}
{"id": "e6d172bd-01f6-4eee-8b59-bb3aa83fb34a", "text": "牧尘测试记忆2026 ", "facts": [], "category": "episodes", "timestamp": "2026-05-25T09:14:35.517132", "tier": "normal", "agent_id": "hermes"}
{"id": "4980e5cc-47de-4317-be4c-99291a2112b5", "text": "牧尘测试记忆2026-05-25 ", "facts": [], "category": "episodes", "timestamp": "2026-05-25T09:15:17.314559", "tier": "normal", "agent_id": "hermes"}

Binary file not shown.

View File

@ -1 +1 @@
{"model_name": "bge-m3", "doc_count": 491, "vector_dim": 1024, "index_type": "FlatIP", "updated_at": "2026-05-26T13:48:13.689259"}
{"model_name": "bge-m3", "doc_count": 492, "vector_dim": 1024, "index_type": "FlatIP", "updated_at": "2026-05-26T14:44:54.107976"}

View File

@ -25,5 +25,5 @@ def get_config():
'decay_rate': 0.015,
'decay_floor': 0.1,
'distill_threshold': 0.7,
'version': '2.6'
'version': '3.0'
}

View File

@ -128,9 +128,33 @@ def commit_memory(req: CommitRequest):
distilled_id = None
try:
distilled = _engine.distill_episode(episode)
_store.append('distilled', distilled.to_dict())
distilled_id = distilled.id
# 冗余消除:自动合并相似度 > 0.8 的记忆(保留最新)
try:
new_summary = distilled.summary
new_content_prefix = (distilled.summary or distilled.to_dict().get('content', '')[:80])
for record in _store.search_recent('distilled', '', months_back=1, limit=200):
if record.get('id') == distilled.id:
continue
old_prefix = (record.get('summary') or record.get('content', ''))[:80]
if not old_prefix or not new_content_prefix:
continue
# overlap coefficient
old_chars = set(old_prefix)
new_chars = set(new_content_prefix)
intersection = len(old_chars & new_chars)
overlap = intersection / min(len(old_chars), len(new_chars)) if min(len(old_chars), len(new_chars)) > 0 else 0
if overlap > 0.8:
from storage.tombstone import TombstoneStore
TombstoneStore().mark_deleted(record.get('id', ''), reason=f'冗余消除: 与 {distilled.id[:16]} 相似度 {overlap:.2f}')
print(f"[Dedupe] 冗余合并: {record.get('id', '')[:20]}{distilled.id[:20]} (overlap={overlap:.2f})")
break # 每条新记忆最多合并一条旧记录
except Exception as e:
print(f"[Dedupe] 冗余消除失败: {e}")
_store.append('distilled', distilled.to_dict())
# Redis 存储 distilled多实例共享
if redis_store:
try:
@ -140,6 +164,23 @@ def commit_memory(req: CommitRequest):
except Exception as e:
print(f"[Redis] distilled 存储/发布失败: {e}")
# 自动冲突检测commit 后立即执行,无需手动调用 /conflicts/scan
try:
from distill.conflicts import ConflictDetector
detector = ConflictDetector()
all_distilled = list(_store.search_recent('distilled', '', months_back=1, limit=1000))
class FakeDistilled:
def __init__(self, d):
self.id = d.get('id', '')
self.entities = d.get('entities', [])
self.facts = d.get('facts', [])
conflicts = detector.scan_conflicts(FakeDistilled(distilled.to_dict()), all_distilled)
if conflicts:
for c in conflicts:
print(f"[Conflict] 检测到冲突: {c.id} type={c.type} entity={c.entity}")
except Exception as e:
print(f"[Conflict] 冲突检测失败: {e}")
# 全量索引重建(每次 commit 都重建,保证 FAISS 索引和数据完全一致)
# 从 JSONLShardStore 读取完整 episodes + distilled最新状态
from storage.bge_embedder import get_bge_client

View File

@ -68,11 +68,11 @@ app.include_router(graph.router, prefix='/api/v1', tags=['graph'])
@app.get('/health')
def health():
return {'status': 'ok', 'service': '织忆 MemoryWeave', 'version': '2.6'}
return {'status': 'ok', 'service': '织忆 MemoryWeave', 'version': '3.0'}
@app.get('/')
def root():
return {'service': '织忆 MemoryWeave', 'version': '2.6', 'docs': '/docs'}
return {'service': '织忆 MemoryWeave', 'version': '3.0', 'docs': '/docs'}
def main():
import uvicorn

View File

@ -116,11 +116,7 @@ class DistillEngine:
summary=episode.content[:200],
entities=entities,
facts=facts,
confidence=eval_result['overall'],
status='distilled',
importance=1 if eval_result['should_distill'] else 0,
created_at=datetime.now(),
updated_at=datetime.now(),
tier=episode.tier,
agent_id=episode.metadata.get('agent_id', 'hermes'),
)

View File

@ -6,9 +6,6 @@ TYPE_REQUEST = 'request'
TYPE_FACT = 'fact'
TYPE_PATTERN = 'pattern'
STATUS_PENDING = 'pending'
STATUS_VALIDATED = 'validated'
STATUS_DEPRECATED = 'deprecated'
@dataclass
class Distilled:
@ -18,13 +15,10 @@ class Distilled:
summary: str
entities: list[str] = field(default_factory=list)
facts: list[str] = field(default_factory=list)
confidence: float = 0.5
status: str = 'pending'
importance: int = 0
created_at: datetime = field(default_factory=datetime.now)
updated_at: datetime = field(default_factory=datetime.now)
tier: str = 'normal' # 继承自 episodecore=永不衰减
agent_id: str = 'hermes' # 继承自 episode用于隐私隔离
importance: float = 0.0
tier: str = 'normal'
agent_id: str = 'hermes'
metadata: dict = field(default_factory=dict)
def to_dict(self) -> dict:
return {
@ -34,20 +28,18 @@ class Distilled:
'summary': self.summary,
'entities': self.entities,
'facts': self.facts,
'confidence': self.confidence,
'status': self.status,
'importance': self.importance,
'created_at': self.created_at.isoformat(),
'updated_at': self.updated_at.isoformat(),
'tier': self.tier,
'agent_id': self.agent_id,
'metadata': self.metadata,
}
@classmethod
def from_dict(cls, d: dict):
d = d.copy()
d['created_at'] = datetime.fromisoformat(d['created_at'])
d['updated_at'] = datetime.fromisoformat(d['updated_at'])
# 兼容旧记录(可能含已删除字段)
for old_field in ('confidence', 'status', 'created_at', 'updated_at'):
d.pop(old_field, None)
if 'tier' not in d:
d['tier'] = 'normal'
if 'agent_id' not in d: