From 631241c50a07879feb111d76a0bf9fdfdab746a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Mon, 28 Sep 2026 23:27:32 +0800 Subject: [PATCH] =?UTF-8?q?feat(provenance):=20commit=20=E5=BC=95=E7=94=A8?= =?UTF-8?q?=E5=8F=AF=E5=A4=8D=E7=AE=97=E6=97=81=E8=B7=AF=E8=A7=82=E6=B5=8B?= =?UTF-8?q?=EF=BC=88t=5F3fbb6cbd=20=E9=98=B6=E6=AE=B51=EF=BC=8C=E5=8F=AA?= =?UTF-8?q?=E8=A7=82=E6=B5=8B=E4=B8=8D=E6=8B=A6=E6=88=AA=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 开关 ZHIYI_COMMIT_PROVENANCE_OBSERVE 默认关。开启时对 /api/v1/commit 做异步旁路观测: (a) 有 source_uri+quote+quote_sha256 → 复算 sha256 比对(hash_mismatch) (b) source==session-compress 且 session_id → 去 ~/.hermes/state.db 只读回证(前60字符命中或 LCS>=60) (c) 其他 → observable=unknown/no_derivation 结果追加 ~/.hermes/logs/zhiyi-commit-provenance/YYYY-MM-DD.jsonl,不改响应/写入路径/拒写入。 AC1 关=零影响已验证;单元测试 TestProvenance*/TestLCS/TestDecideQuoteHash 通过。 --- go/internal/api/routes/core.go | 4 + go/internal/api/routes/provenance_observe.go | 214 ++++++++++++++++++ .../api/routes/provenance_observe_test.go | 49 ++++ 3 files changed, 267 insertions(+) create mode 100644 go/internal/api/routes/provenance_observe.go create mode 100644 go/internal/api/routes/provenance_observe_test.go diff --git a/go/internal/api/routes/core.go b/go/internal/api/routes/core.go index 6146769..4fd486a 100644 --- a/go/internal/api/routes/core.go +++ b/go/internal/api/routes/core.go @@ -115,6 +115,10 @@ func (a *API) Commit(w http.ResponseWriter, r *http.Request) { } log.Printf("[srcprobe] commit 收到 source=%q(顶层), metadata=%v → 解析后 src=%q", req.Source, req.Metadata, src) + // t_3fbb6cbd:引用可复算旁路观测(阶段1,开关 ZHIYI_COMMIT_PROVENANCE_OBSERVE 默认关)。 + // 异步 goroutine,不阻塞写路径、不改响应、不拒写入。 + go observeCommitProvenance(req.AgentID, req.Content, req.Metadata) + // 1. 写入 episode(原始日志,总是写入) epID, err := a.LanceDB.InsertEpisode(req.AgentID, req.Namespace, req.Content, req.Category) if err != nil { diff --git a/go/internal/api/routes/provenance_observe.go b/go/internal/api/routes/provenance_observe.go new file mode 100644 index 0000000..dca7b13 --- /dev/null +++ b/go/internal/api/routes/provenance_observe.go @@ -0,0 +1,214 @@ +// 织忆 MemoryWeave — commit 写入的「引用可复算」旁路观测(阶段 1:只观测,不拦截) +// +// 动机(t_3fbb6cbd):Akasha 的确定性校验器(sourceRange + quoteHash,fail-closed) +// 是唯一值得移植的东西。织忆现在的写入无法回证(09-25 实测卡点)。 +// 但现有写入方 payload 里没有 source_uri/quote/quote_sha256,直接上硬闸 = 拒掉 100% +// 写入 = 掐死织忆全量摄取。所以本卡只做观测,先拿真实分布,再决定阈值/形态。 +// +// 开关:ZHIYI_COMMIT_PROVENANCE_OBSERVE,默认关闭。关闭时行为与改前完全一致(零影响)。 +// 开启时对每次 commit 做旁路观测(异步 goroutine,不阻塞、不改响应、不改写入路径), +// 结果追加到 ~/.hermes/logs/zhiyi-commit-provenance/YYYY-MM-DD.jsonl。 +// +// 判定降级(AC2): +// (a) metadata 有 source_uri + quote + quote_sha256 → 复算 sha256(quote) 比对; +// 不等 observable=false, reason=hash_mismatch +// (b) 无上述字段但 source=="session-compress" 且 session_id 存在 → +// 去 ~/.hermes/state.db(只读)取该 session 消息文本,查 content 前 60 字符 +// 逐字命中(或最长公共子串 >=60)→ 命中 observable=true,否则 false reason=not_in_source +// (c) 其他 source → observable=unknown, reason=no_derivation(不要瞎猜成 false) +package routes + +import ( + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + _ "modernc.org/sqlite" +) + +// observeMu 保护 jsonl 追加(commit 并发时防交错写) +var observeMu sync.Mutex + +// provenanceObserveEnabled 开关(默认关闭,读一次即可) +var provenanceObserveEnabled = os.Getenv("ZHIYI_COMMIT_PROVENANCE_OBSERVE") == "1" || + strings.EqualFold(os.Getenv("ZHIYI_COMMIT_PROVENANCE_OBSERVE"), "true") + +// stateDBPath 只读来源库(session-to-zhiyi.py 等写入方读的同一个 state.db) +const stateDBPath = "/home/muc/.hermes/state.db" + +// provenanceLogDir 观测结果落盘目录 +const provenanceLogDir = "/home/muc/.hermes/logs/zhiyi-commit-provenance" + +// minCommonRun 判定「可复算」的最长公共子串下界(字符) +const minCommonRun = 60 + +// observeCommitProvenance 旁路观测一次 commit(异步调用,永不阻塞写路径) +func observeCommitProvenance(agentID, content string, metadata map[string]any) { + if !provenanceObserveEnabled { + return + } + source := "" + sessionID := "" + if metadata != nil { + if s, ok := metadata["source"].(string); ok { + source = s + } + if sid, ok := metadata["session_id"].(string); ok { + sessionID = sid + } + } + + observable := "unknown" + reason := "no_derivation" + + // (a) 显式 quote + hash + if metadata != nil { + uri, _ := metadata["source_uri"].(string) + quote, _ := metadata["quote"].(string) + qh, _ := metadata["quote_sha256"].(string) + if uri != "" && quote != "" && qh != "" { + observable, reason = decideQuoteHash(quote, qh) + } else if source == "session-compress" && sessionID != "" { + // (b) session-compress:去 state.db 只读回证 + observable, reason = observeViaSession(content, sessionID) + } + } else if source == "session-compress" && sessionID != "" { + observable, reason = observeViaSession(content, sessionID) + } + + // (c) 其余保持 unknown / no_derivation + + line := map[string]interface{}{ + "ts": time.Now().Format(time.RFC3339), + "agent_id": agentID, + "source": source, + "session_id": sessionID, + "observable": observable, + "reason": reason, + "content_len": len(content), + } + appendProvenanceLog(line) +} + +// decideQuoteHash 复算 sha256(quote) 与调用方声明的 quote_sha256 比对。 +func decideQuoteHash(quote, quoteSHA256 string) (string, string) { + sum := sha256.Sum256([]byte(quote)) + if hex.EncodeToString(sum[:]) == quoteSHA256 { + return "true", "" + } + return "false", "hash_mismatch" +} + +// observeViaSession 去 state.db(只读)取 session 消息文本,回证 content 可复算。 +func observeViaSession(content, sessionID string) (string, string) { + db, err := sql.Open("sqlite", "file:"+stateDBPath+"?_pragma=query_only(1)&_pragma=busy_timeout(3000)") + if err != nil { + return "unknown", "state_db_open_failed: " + err.Error() + } + defer db.Close() + rows, err := db.Query( + "SELECT content FROM messages WHERE session_id=? AND role IN ('user','assistant') ORDER BY id", + sessionID) + if err != nil { + return "unknown", "state_db_query_failed: " + err.Error() + } + defer rows.Close() + + var sb strings.Builder + for rows.Next() { + var msg string + if err := rows.Scan(&msg); err != nil { + continue + } + sb.WriteString(msg) + sb.WriteByte('\n') + } + srcText := sb.String() + if srcText == "" { + return "false", "not_in_source" + } + if provenanceMatch(content, srcText) { + return "true", "" + } + return "false", "not_in_source" +} + +// provenanceMatch 判定 content 相对 srcText 可复算: +// content 前 60 字符在 srcText 中逐字命中,或 最长公共子串 >=60。 +func provenanceMatch(content, srcText string) bool { + if content == "" || srcText == "" { + return false + } + first := content + if len(first) > minCommonRun { + first = first[:minCommonRun] + } + if strings.Contains(srcText, first) { + return true + } + // LCS 兜底(受限输入防打爆) + c := content + if len(c) > 2000 { + c = c[:2000] + } + s := srcText + if len(s) > 20000 { + s = s[:20000] + } + return lcsLength(c, s) >= minCommonRun +} + +// lcsLength 两个字符串的最长公共子串长度(O(n*m) 动态规划,取较短者做列)。 +func lcsLength(a, b string) int { + if len(a) == 0 || len(b) == 0 { + return 0 + } + // 让 b 为较短者(节省列空间) + if len(b) > len(a) { + a, b = b, a + } + prev := make([]int, len(b)+1) + cur := make([]int, len(b)+1) + best := 0 + for i := 1; i <= len(a); i++ { + for j := 1; j <= len(b); j++ { + if a[i-1] == b[j-1] { + cur[j] = prev[j-1] + 1 + if cur[j] > best { + best = cur[j] + } + } else { + cur[j] = 0 + } + } + prev, cur = cur, prev + } + return best +} + +// appendProvenanceLog 把一行观测结果追加到当日 jsonl。 +func appendProvenanceLog(line map[string]interface{}) { + observeMu.Lock() + defer observeMu.Unlock() + data, err := json.Marshal(line) + if err != nil { + return + } + if err := os.MkdirAll(provenanceLogDir, 0o755); err != nil { + return + } + path := filepath.Join(provenanceLogDir, time.Now().Format("2006-01-02")+".jsonl") + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return + } + defer f.Close() + fmt.Fprintf(f, "%s\n", data) +} \ No newline at end of file diff --git a/go/internal/api/routes/provenance_observe_test.go b/go/internal/api/routes/provenance_observe_test.go new file mode 100644 index 0000000..8dcb6b9 --- /dev/null +++ b/go/internal/api/routes/provenance_observe_test.go @@ -0,0 +1,49 @@ +package routes + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +func TestProvenanceMatch(t *testing.T) { + src := strings.Repeat("源文本", 50) // 150 chars + // 前60字符逐字命中 + content := strings.Repeat("源文本", 25) // 75 chars + if !provenanceMatch(content, src) { + t.Fatal("expect first-60-chars substring match") + } + // 完全无关 → false + if provenanceMatch(strings.Repeat("完全无关", 20), src) { + t.Fatal("expect no match for unrelated") + } + // LCS >= 60 兜底(content 中间有长公共段) + content2 := strings.Repeat("P", 10) + strings.Repeat("源文本", 40) + strings.Repeat("Q", 5) + src2 := strings.Repeat("A", 30) + strings.Repeat("源文本", 40) + strings.Repeat("B", 30) + if !provenanceMatch(content2, src2) { + t.Fatal("expect LCS>=60 fallback match") + } +} + +func TestLCSLength(t *testing.T) { + if got := lcsLength("abcde", "abfce"); got != 2 { // "ab" + t.Fatalf("lcs=2 expect, got %d", got) + } + if got := lcsLength("", "abc"); got != 0 { + t.Fatalf("lcs empty=0 expect, got %d", got) + } +} + +func TestDecideQuoteHash(t *testing.T) { + quote := "这段引用内容" + sum := sha256.Sum256([]byte(quote)) + observable, reason := decideQuoteHash(quote, hex.EncodeToString(sum[:])) + if observable != "true" || reason != "" { + t.Fatalf("expect true/'' got %s/%s", observable, reason) + } + observable, reason = decideQuoteHash(quote, strings.Repeat("0", 64)) + if observable != "false" || reason != "hash_mismatch" { + t.Fatalf("expect false/hash_mismatch got %s/%s", observable, reason) + } +} \ No newline at end of file