feat(provenance): commit 引用可复算旁路观测(t_3fbb6cbd 阶段1,只观测不拦截)
开关 ZHIYI_COMMIT_PROVENANCE_OBSERVE 默认关。开启时对 /api/v1/commit 做异步旁路观测: (a) 有 source_uri+quote+quote_sha256 → 复算 sha256 比对(hash_mismatch) (b) source==session-compress 且 session_id → 去 ~/.hermes/state.db 只读回证(前60字符命中或 LCS>=60) (c) 其他 → observable=unknown/no_derivation 结果追加 ~/.hermes/logs/zhiyi-commit-provenance/YYYY-MM-DD.jsonl,不改响应/写入路径/拒写入。 AC1 关=零影响已验证;单元测试 TestProvenance*/TestLCS/TestDecideQuoteHash 通过。
This commit is contained in:
@@ -115,6 +115,10 @@ func (a *API) Commit(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
log.Printf("[srcprobe] commit 收到 source=%q(顶层), metadata=%v → 解析后 src=%q", req.Source, req.Metadata, src)
|
||||
|
||||
// t_3fbb6cbd:引用可复算旁路观测(阶段1,开关 ZHIYI_COMMIT_PROVENANCE_OBSERVE 默认关)。
|
||||
// 异步 goroutine,不阻塞写路径、不改响应、不拒写入。
|
||||
go observeCommitProvenance(req.AgentID, req.Content, req.Metadata)
|
||||
|
||||
// 1. 写入 episode(原始日志,总是写入)
|
||||
epID, err := a.LanceDB.InsertEpisode(req.AgentID, req.Namespace, req.Content, req.Category)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
// 织忆 MemoryWeave — commit 写入的「引用可复算」旁路观测(阶段 1:只观测,不拦截)
|
||||
//
|
||||
// 动机(t_3fbb6cbd):Akasha 的确定性校验器(sourceRange + quoteHash,fail-closed)
|
||||
// 是唯一值得移植的东西。织忆现在的写入无法回证(09-25 实测卡点)。
|
||||
// 但现有写入方 payload 里没有 source_uri/quote/quote_sha256,直接上硬闸 = 拒掉 100%
|
||||
// 写入 = 掐死织忆全量摄取。所以本卡只做观测,先拿真实分布,再决定阈值/形态。
|
||||
//
|
||||
// 开关:ZHIYI_COMMIT_PROVENANCE_OBSERVE,默认关闭。关闭时行为与改前完全一致(零影响)。
|
||||
// 开启时对每次 commit 做旁路观测(异步 goroutine,不阻塞、不改响应、不改写入路径),
|
||||
// 结果追加到 ~/.hermes/logs/zhiyi-commit-provenance/YYYY-MM-DD.jsonl。
|
||||
//
|
||||
// 判定降级(AC2):
|
||||
// (a) metadata 有 source_uri + quote + quote_sha256 → 复算 sha256(quote) 比对;
|
||||
// 不等 observable=false, reason=hash_mismatch
|
||||
// (b) 无上述字段但 source=="session-compress" 且 session_id 存在 →
|
||||
// 去 ~/.hermes/state.db(只读)取该 session 消息文本,查 content 前 60 字符
|
||||
// 逐字命中(或最长公共子串 >=60)→ 命中 observable=true,否则 false reason=not_in_source
|
||||
// (c) 其他 source → observable=unknown, reason=no_derivation(不要瞎猜成 false)
|
||||
package routes
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// observeMu 保护 jsonl 追加(commit 并发时防交错写)
|
||||
var observeMu sync.Mutex
|
||||
|
||||
// provenanceObserveEnabled 开关(默认关闭,读一次即可)
|
||||
var provenanceObserveEnabled = os.Getenv("ZHIYI_COMMIT_PROVENANCE_OBSERVE") == "1" ||
|
||||
strings.EqualFold(os.Getenv("ZHIYI_COMMIT_PROVENANCE_OBSERVE"), "true")
|
||||
|
||||
// stateDBPath 只读来源库(session-to-zhiyi.py 等写入方读的同一个 state.db)
|
||||
const stateDBPath = "/home/muc/.hermes/state.db"
|
||||
|
||||
// provenanceLogDir 观测结果落盘目录
|
||||
const provenanceLogDir = "/home/muc/.hermes/logs/zhiyi-commit-provenance"
|
||||
|
||||
// minCommonRun 判定「可复算」的最长公共子串下界(字符)
|
||||
const minCommonRun = 60
|
||||
|
||||
// observeCommitProvenance 旁路观测一次 commit(异步调用,永不阻塞写路径)
|
||||
func observeCommitProvenance(agentID, content string, metadata map[string]any) {
|
||||
if !provenanceObserveEnabled {
|
||||
return
|
||||
}
|
||||
source := ""
|
||||
sessionID := ""
|
||||
if metadata != nil {
|
||||
if s, ok := metadata["source"].(string); ok {
|
||||
source = s
|
||||
}
|
||||
if sid, ok := metadata["session_id"].(string); ok {
|
||||
sessionID = sid
|
||||
}
|
||||
}
|
||||
|
||||
observable := "unknown"
|
||||
reason := "no_derivation"
|
||||
|
||||
// (a) 显式 quote + hash
|
||||
if metadata != nil {
|
||||
uri, _ := metadata["source_uri"].(string)
|
||||
quote, _ := metadata["quote"].(string)
|
||||
qh, _ := metadata["quote_sha256"].(string)
|
||||
if uri != "" && quote != "" && qh != "" {
|
||||
observable, reason = decideQuoteHash(quote, qh)
|
||||
} else if source == "session-compress" && sessionID != "" {
|
||||
// (b) session-compress:去 state.db 只读回证
|
||||
observable, reason = observeViaSession(content, sessionID)
|
||||
}
|
||||
} else if source == "session-compress" && sessionID != "" {
|
||||
observable, reason = observeViaSession(content, sessionID)
|
||||
}
|
||||
|
||||
// (c) 其余保持 unknown / no_derivation
|
||||
|
||||
line := map[string]interface{}{
|
||||
"ts": time.Now().Format(time.RFC3339),
|
||||
"agent_id": agentID,
|
||||
"source": source,
|
||||
"session_id": sessionID,
|
||||
"observable": observable,
|
||||
"reason": reason,
|
||||
"content_len": len(content),
|
||||
}
|
||||
appendProvenanceLog(line)
|
||||
}
|
||||
|
||||
// decideQuoteHash 复算 sha256(quote) 与调用方声明的 quote_sha256 比对。
|
||||
func decideQuoteHash(quote, quoteSHA256 string) (string, string) {
|
||||
sum := sha256.Sum256([]byte(quote))
|
||||
if hex.EncodeToString(sum[:]) == quoteSHA256 {
|
||||
return "true", ""
|
||||
}
|
||||
return "false", "hash_mismatch"
|
||||
}
|
||||
|
||||
// observeViaSession 去 state.db(只读)取 session 消息文本,回证 content 可复算。
|
||||
func observeViaSession(content, sessionID string) (string, string) {
|
||||
db, err := sql.Open("sqlite", "file:"+stateDBPath+"?_pragma=query_only(1)&_pragma=busy_timeout(3000)")
|
||||
if err != nil {
|
||||
return "unknown", "state_db_open_failed: " + err.Error()
|
||||
}
|
||||
defer db.Close()
|
||||
rows, err := db.Query(
|
||||
"SELECT content FROM messages WHERE session_id=? AND role IN ('user','assistant') ORDER BY id",
|
||||
sessionID)
|
||||
if err != nil {
|
||||
return "unknown", "state_db_query_failed: " + err.Error()
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var sb strings.Builder
|
||||
for rows.Next() {
|
||||
var msg string
|
||||
if err := rows.Scan(&msg); err != nil {
|
||||
continue
|
||||
}
|
||||
sb.WriteString(msg)
|
||||
sb.WriteByte('\n')
|
||||
}
|
||||
srcText := sb.String()
|
||||
if srcText == "" {
|
||||
return "false", "not_in_source"
|
||||
}
|
||||
if provenanceMatch(content, srcText) {
|
||||
return "true", ""
|
||||
}
|
||||
return "false", "not_in_source"
|
||||
}
|
||||
|
||||
// provenanceMatch 判定 content 相对 srcText 可复算:
|
||||
// content 前 60 字符在 srcText 中逐字命中,或 最长公共子串 >=60。
|
||||
func provenanceMatch(content, srcText string) bool {
|
||||
if content == "" || srcText == "" {
|
||||
return false
|
||||
}
|
||||
first := content
|
||||
if len(first) > minCommonRun {
|
||||
first = first[:minCommonRun]
|
||||
}
|
||||
if strings.Contains(srcText, first) {
|
||||
return true
|
||||
}
|
||||
// LCS 兜底(受限输入防打爆)
|
||||
c := content
|
||||
if len(c) > 2000 {
|
||||
c = c[:2000]
|
||||
}
|
||||
s := srcText
|
||||
if len(s) > 20000 {
|
||||
s = s[:20000]
|
||||
}
|
||||
return lcsLength(c, s) >= minCommonRun
|
||||
}
|
||||
|
||||
// lcsLength 两个字符串的最长公共子串长度(O(n*m) 动态规划,取较短者做列)。
|
||||
func lcsLength(a, b string) int {
|
||||
if len(a) == 0 || len(b) == 0 {
|
||||
return 0
|
||||
}
|
||||
// 让 b 为较短者(节省列空间)
|
||||
if len(b) > len(a) {
|
||||
a, b = b, a
|
||||
}
|
||||
prev := make([]int, len(b)+1)
|
||||
cur := make([]int, len(b)+1)
|
||||
best := 0
|
||||
for i := 1; i <= len(a); i++ {
|
||||
for j := 1; j <= len(b); j++ {
|
||||
if a[i-1] == b[j-1] {
|
||||
cur[j] = prev[j-1] + 1
|
||||
if cur[j] > best {
|
||||
best = cur[j]
|
||||
}
|
||||
} else {
|
||||
cur[j] = 0
|
||||
}
|
||||
}
|
||||
prev, cur = cur, prev
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// appendProvenanceLog 把一行观测结果追加到当日 jsonl。
|
||||
func appendProvenanceLog(line map[string]interface{}) {
|
||||
observeMu.Lock()
|
||||
defer observeMu.Unlock()
|
||||
data, err := json.Marshal(line)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(provenanceLogDir, 0o755); err != nil {
|
||||
return
|
||||
}
|
||||
path := filepath.Join(provenanceLogDir, time.Now().Format("2006-01-02")+".jsonl")
|
||||
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
fmt.Fprintf(f, "%s\n", data)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package routes
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProvenanceMatch(t *testing.T) {
|
||||
src := strings.Repeat("源文本", 50) // 150 chars
|
||||
// 前60字符逐字命中
|
||||
content := strings.Repeat("源文本", 25) // 75 chars
|
||||
if !provenanceMatch(content, src) {
|
||||
t.Fatal("expect first-60-chars substring match")
|
||||
}
|
||||
// 完全无关 → false
|
||||
if provenanceMatch(strings.Repeat("完全无关", 20), src) {
|
||||
t.Fatal("expect no match for unrelated")
|
||||
}
|
||||
// LCS >= 60 兜底(content 中间有长公共段)
|
||||
content2 := strings.Repeat("P", 10) + strings.Repeat("源文本", 40) + strings.Repeat("Q", 5)
|
||||
src2 := strings.Repeat("A", 30) + strings.Repeat("源文本", 40) + strings.Repeat("B", 30)
|
||||
if !provenanceMatch(content2, src2) {
|
||||
t.Fatal("expect LCS>=60 fallback match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLCSLength(t *testing.T) {
|
||||
if got := lcsLength("abcde", "abfce"); got != 2 { // "ab"
|
||||
t.Fatalf("lcs=2 expect, got %d", got)
|
||||
}
|
||||
if got := lcsLength("", "abc"); got != 0 {
|
||||
t.Fatalf("lcs empty=0 expect, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideQuoteHash(t *testing.T) {
|
||||
quote := "这段引用内容"
|
||||
sum := sha256.Sum256([]byte(quote))
|
||||
observable, reason := decideQuoteHash(quote, hex.EncodeToString(sum[:]))
|
||||
if observable != "true" || reason != "" {
|
||||
t.Fatalf("expect true/'' got %s/%s", observable, reason)
|
||||
}
|
||||
observable, reason = decideQuoteHash(quote, strings.Repeat("0", 64))
|
||||
if observable != "false" || reason != "hash_mismatch" {
|
||||
t.Fatalf("expect false/hash_mismatch got %s/%s", observable, reason)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user