feat(provenance): commit 引用可复算旁路观测(t_3fbb6cbd 阶段1,只观测不拦截)

开关 ZHIYI_COMMIT_PROVENANCE_OBSERVE 默认关。开启时对 /api/v1/commit 做异步旁路观测:
(a) 有 source_uri+quote+quote_sha256 → 复算 sha256 比对(hash_mismatch)
(b) source==session-compress 且 session_id → 去 ~/.hermes/state.db 只读回证(前60字符命中或 LCS>=60)
(c) 其他 → observable=unknown/no_derivation
结果追加 ~/.hermes/logs/zhiyi-commit-provenance/YYYY-MM-DD.jsonl,不改响应/写入路径/拒写入。
AC1 关=零影响已验证;单元测试 TestProvenance*/TestLCS/TestDecideQuoteHash 通过。
This commit is contained in:
小唯
2026-09-28 23:27:32 +08:00
parent 66cb258979
commit 631241c50a
3 changed files with 267 additions and 0 deletions
+4
View File
@@ -115,6 +115,10 @@ func (a *API) Commit(w http.ResponseWriter, r *http.Request) {
}
log.Printf("[srcprobe] commit 收到 source=%q(顶层), metadata=%v → 解析后 src=%q", req.Source, req.Metadata, src)
// t_3fbb6cbd:引用可复算旁路观测(阶段1,开关 ZHIYI_COMMIT_PROVENANCE_OBSERVE 默认关)。
// 异步 goroutine,不阻塞写路径、不改响应、不拒写入。
go observeCommitProvenance(req.AgentID, req.Content, req.Metadata)
// 1. 写入 episode(原始日志,总是写入)
epID, err := a.LanceDB.InsertEpisode(req.AgentID, req.Namespace, req.Content, req.Category)
if err != nil {
@@ -0,0 +1,214 @@
// 织忆 MemoryWeave — commit 写入的「引用可复算」旁路观测(阶段 1:只观测,不拦截)
//
// 动机(t_3fbb6cbd):Akasha 的确定性校验器(sourceRange + quoteHash,fail-closed)
// 是唯一值得移植的东西。织忆现在的写入无法回证(09-25 实测卡点)。
// 但现有写入方 payload 里没有 source_uri/quote/quote_sha256,直接上硬闸 = 拒掉 100%
// 写入 = 掐死织忆全量摄取。所以本卡只做观测,先拿真实分布,再决定阈值/形态。
//
// 开关:ZHIYI_COMMIT_PROVENANCE_OBSERVE,默认关闭。关闭时行为与改前完全一致(零影响)。
// 开启时对每次 commit 做旁路观测(异步 goroutine,不阻塞、不改响应、不改写入路径),
// 结果追加到 ~/.hermes/logs/zhiyi-commit-provenance/YYYY-MM-DD.jsonl。
//
// 判定降级(AC2):
// (a) metadata 有 source_uri + quote + quote_sha256 → 复算 sha256(quote) 比对;
// 不等 observable=false, reason=hash_mismatch
// (b) 无上述字段但 source=="session-compress" 且 session_id 存在 →
// 去 ~/.hermes/state.db(只读)取该 session 消息文本,查 content 前 60 字符
// 逐字命中(或最长公共子串 >=60)→ 命中 observable=true,否则 false reason=not_in_source
// (c) 其他 source → observable=unknown, reason=no_derivation(不要瞎猜成 false)
package routes
import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
_ "modernc.org/sqlite"
)
// observeMu 保护 jsonl 追加(commit 并发时防交错写)
var observeMu sync.Mutex
// provenanceObserveEnabled 开关(默认关闭,读一次即可)
var provenanceObserveEnabled = os.Getenv("ZHIYI_COMMIT_PROVENANCE_OBSERVE") == "1" ||
strings.EqualFold(os.Getenv("ZHIYI_COMMIT_PROVENANCE_OBSERVE"), "true")
// stateDBPath 只读来源库(session-to-zhiyi.py 等写入方读的同一个 state.db)
const stateDBPath = "/home/muc/.hermes/state.db"
// provenanceLogDir 观测结果落盘目录
const provenanceLogDir = "/home/muc/.hermes/logs/zhiyi-commit-provenance"
// minCommonRun 判定「可复算」的最长公共子串下界(字符)
const minCommonRun = 60
// observeCommitProvenance 旁路观测一次 commit(异步调用,永不阻塞写路径)
func observeCommitProvenance(agentID, content string, metadata map[string]any) {
if !provenanceObserveEnabled {
return
}
source := ""
sessionID := ""
if metadata != nil {
if s, ok := metadata["source"].(string); ok {
source = s
}
if sid, ok := metadata["session_id"].(string); ok {
sessionID = sid
}
}
observable := "unknown"
reason := "no_derivation"
// (a) 显式 quote + hash
if metadata != nil {
uri, _ := metadata["source_uri"].(string)
quote, _ := metadata["quote"].(string)
qh, _ := metadata["quote_sha256"].(string)
if uri != "" && quote != "" && qh != "" {
observable, reason = decideQuoteHash(quote, qh)
} else if source == "session-compress" && sessionID != "" {
// (b) session-compress:去 state.db 只读回证
observable, reason = observeViaSession(content, sessionID)
}
} else if source == "session-compress" && sessionID != "" {
observable, reason = observeViaSession(content, sessionID)
}
// (c) 其余保持 unknown / no_derivation
line := map[string]interface{}{
"ts": time.Now().Format(time.RFC3339),
"agent_id": agentID,
"source": source,
"session_id": sessionID,
"observable": observable,
"reason": reason,
"content_len": len(content),
}
appendProvenanceLog(line)
}
// decideQuoteHash 复算 sha256(quote) 与调用方声明的 quote_sha256 比对。
func decideQuoteHash(quote, quoteSHA256 string) (string, string) {
sum := sha256.Sum256([]byte(quote))
if hex.EncodeToString(sum[:]) == quoteSHA256 {
return "true", ""
}
return "false", "hash_mismatch"
}
// observeViaSession 去 state.db(只读)取 session 消息文本,回证 content 可复算。
func observeViaSession(content, sessionID string) (string, string) {
db, err := sql.Open("sqlite", "file:"+stateDBPath+"?_pragma=query_only(1)&_pragma=busy_timeout(3000)")
if err != nil {
return "unknown", "state_db_open_failed: " + err.Error()
}
defer db.Close()
rows, err := db.Query(
"SELECT content FROM messages WHERE session_id=? AND role IN ('user','assistant') ORDER BY id",
sessionID)
if err != nil {
return "unknown", "state_db_query_failed: " + err.Error()
}
defer rows.Close()
var sb strings.Builder
for rows.Next() {
var msg string
if err := rows.Scan(&msg); err != nil {
continue
}
sb.WriteString(msg)
sb.WriteByte('\n')
}
srcText := sb.String()
if srcText == "" {
return "false", "not_in_source"
}
if provenanceMatch(content, srcText) {
return "true", ""
}
return "false", "not_in_source"
}
// provenanceMatch 判定 content 相对 srcText 可复算:
// content 前 60 字符在 srcText 中逐字命中,或 最长公共子串 >=60。
func provenanceMatch(content, srcText string) bool {
if content == "" || srcText == "" {
return false
}
first := content
if len(first) > minCommonRun {
first = first[:minCommonRun]
}
if strings.Contains(srcText, first) {
return true
}
// LCS 兜底(受限输入防打爆)
c := content
if len(c) > 2000 {
c = c[:2000]
}
s := srcText
if len(s) > 20000 {
s = s[:20000]
}
return lcsLength(c, s) >= minCommonRun
}
// lcsLength 两个字符串的最长公共子串长度(O(n*m) 动态规划,取较短者做列)。
func lcsLength(a, b string) int {
if len(a) == 0 || len(b) == 0 {
return 0
}
// 让 b 为较短者(节省列空间)
if len(b) > len(a) {
a, b = b, a
}
prev := make([]int, len(b)+1)
cur := make([]int, len(b)+1)
best := 0
for i := 1; i <= len(a); i++ {
for j := 1; j <= len(b); j++ {
if a[i-1] == b[j-1] {
cur[j] = prev[j-1] + 1
if cur[j] > best {
best = cur[j]
}
} else {
cur[j] = 0
}
}
prev, cur = cur, prev
}
return best
}
// appendProvenanceLog 把一行观测结果追加到当日 jsonl。
func appendProvenanceLog(line map[string]interface{}) {
observeMu.Lock()
defer observeMu.Unlock()
data, err := json.Marshal(line)
if err != nil {
return
}
if err := os.MkdirAll(provenanceLogDir, 0o755); err != nil {
return
}
path := filepath.Join(provenanceLogDir, time.Now().Format("2006-01-02")+".jsonl")
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
return
}
defer f.Close()
fmt.Fprintf(f, "%s\n", data)
}
@@ -0,0 +1,49 @@
package routes
import (
"crypto/sha256"
"encoding/hex"
"strings"
"testing"
)
func TestProvenanceMatch(t *testing.T) {
src := strings.Repeat("源文本", 50) // 150 chars
// 前60字符逐字命中
content := strings.Repeat("源文本", 25) // 75 chars
if !provenanceMatch(content, src) {
t.Fatal("expect first-60-chars substring match")
}
// 完全无关 → false
if provenanceMatch(strings.Repeat("完全无关", 20), src) {
t.Fatal("expect no match for unrelated")
}
// LCS >= 60 兜底(content 中间有长公共段)
content2 := strings.Repeat("P", 10) + strings.Repeat("源文本", 40) + strings.Repeat("Q", 5)
src2 := strings.Repeat("A", 30) + strings.Repeat("源文本", 40) + strings.Repeat("B", 30)
if !provenanceMatch(content2, src2) {
t.Fatal("expect LCS>=60 fallback match")
}
}
func TestLCSLength(t *testing.T) {
if got := lcsLength("abcde", "abfce"); got != 2 { // "ab"
t.Fatalf("lcs=2 expect, got %d", got)
}
if got := lcsLength("", "abc"); got != 0 {
t.Fatalf("lcs empty=0 expect, got %d", got)
}
}
func TestDecideQuoteHash(t *testing.T) {
quote := "这段引用内容"
sum := sha256.Sum256([]byte(quote))
observable, reason := decideQuoteHash(quote, hex.EncodeToString(sum[:]))
if observable != "true" || reason != "" {
t.Fatalf("expect true/'' got %s/%s", observable, reason)
}
observable, reason = decideQuoteHash(quote, strings.Repeat("0", 64))
if observable != "false" || reason != "hash_mismatch" {
t.Fatalf("expect false/hash_mismatch got %s/%s", observable, reason)
}
}