背景:not_useful_count 被两种语义混写——用户人工点「没用」(feedback.go MarkNotUseful) 与 dedup/合并流程自动降权(core.go Dedup demote) 都写同一字段。 实测 35010 行里 509 条非零但 human useful 仅 21 ⇒ 字段被混写。P2 自适应阈值 要靠它当标签,统计即垃圾。 改法(AC1-5,禁部署): - LanceDB(arrow_schema) + sqlite(create/alter) 两侧同步新增 human_not_useful_count / dedup_downweight_count(additive,不删旧列) - 两个写入点各写各的:feedback.go→IncrementHumanNotUseful; core.go dedup→IncrementDedupDownweight - not_useful_count 保留 deprecated,新代码不再读(读侧仅 Scan/迁移保留) - 存量迁移工具 cmd/split_notuseful(--dry-run 默认,apply 需终审); 实测存量无可按来源判定行(M=0)→ 全部归 unknown(K=109 存活行),apply=no-op - 单测 TestSplitNotUsefulFieldIsolation:两写入点写不同字段,相对增量断言 go build/vet/test + rust build 全绿;rust open() schema 演进在副本上 实测补列 human_not_useful_count/dedup_downweight_count 成功。
172 lines
5.4 KiB
Go
172 lines
5.4 KiB
Go
// 织忆 MemoryWeave — not_useful_count 拆分迁移工具(GA-P0-1 t_718d8707)
|
||
//
|
||
// 背景:not_useful_count 被两种语义混写——用户人工点「没用」(feedback.go
|
||
// MarkNotUseful)与 dedup/合并流程自动降权(core.go Dedup demote)都写同一个字段。
|
||
// 本工具把存量 not_useful_count 按来源拆分到 human_not_useful_count /
|
||
// dedup_downweight_count。
|
||
//
|
||
// 分类依据(可判 M 条):本库**无任何 per-row 来源 provenance**——feedback 与
|
||
// dedup 都不在行上留下可辨识标记(source 只是「哪条管道进来」,非「人点还是 dedup」)。
|
||
// 故对存量行无法按来源判定:M=0,全部归 unknown(K=N)。apply 在 M=0 时是 no-op,
|
||
// 不会改动任何存量数据(保守安全)。
|
||
//
|
||
// 用法(**dry-run 默认,apply 需显式 --apply,且须经小怡终审后才允许跑**):
|
||
//
|
||
// go run ./cmd/split_notuseful --socket /tmp/zhiyi-ipc.sock # dry-run
|
||
// go run ./cmd/split_notuseful --socket /tmp/zhiyi-ipc.sock --apply # 真写(禁未审批)
|
||
package main
|
||
|
||
import (
|
||
"encoding/binary"
|
||
"encoding/json"
|
||
"flag"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"os"
|
||
"sort"
|
||
)
|
||
|
||
type ipcReq struct {
|
||
Type string `json:"type"`
|
||
}
|
||
|
||
type ipcResp struct {
|
||
Status string `json:"status"`
|
||
ReportJSON string `json:"report_json"`
|
||
ErrorDetail string `json:"error_detail"`
|
||
}
|
||
|
||
func rpc(sock string, req ipcReq) (*ipcResp, error) {
|
||
conn, err := net.DialTimeout("unix", sock, 5e9)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("ipc dial %s: %w", sock, err)
|
||
}
|
||
defer conn.Close()
|
||
body, _ := json.Marshal(req)
|
||
lenBuf := make([]byte, 4)
|
||
binary.BigEndian.PutUint32(lenBuf, uint32(len(body)))
|
||
conn.Write(lenBuf)
|
||
conn.Write(body)
|
||
io.ReadFull(conn, lenBuf)
|
||
msgLen := binary.BigEndian.Uint32(lenBuf)
|
||
respBuf := make([]byte, msgLen)
|
||
io.ReadFull(conn, respBuf)
|
||
var resp ipcResp
|
||
if err := json.Unmarshal(respBuf, &resp); err != nil {
|
||
return nil, err
|
||
}
|
||
if resp.Status != "ok" {
|
||
return nil, fmt.Errorf("ipc: %s", resp.ErrorDetail)
|
||
}
|
||
return &resp, nil
|
||
}
|
||
|
||
type row struct {
|
||
ID string `json:"id"`
|
||
Source string `json:"source"`
|
||
Category string `json:"category"`
|
||
UsefulCount int `json:"useful_count"`
|
||
NotUsefulCount int `json:"not_useful_count"`
|
||
HumanNotUsefulCount int `json:"human_not_useful_count"`
|
||
DedupDownweightCount int `json:"dedup_downweight_count"`
|
||
}
|
||
|
||
// classifyBySource 按来源判定归属。返回 "human" / "dedup" / "unknown"。
|
||
// 当前 schema 无 per-row 来源 provenance ⇒ 恒 unknown(M=0)。
|
||
// 将来若某来源可明确标识「人工反馈」或「dedup 降权」,在此追加映射即可。
|
||
func classifyBySource(_ row) string {
|
||
return "unknown"
|
||
}
|
||
|
||
func main() {
|
||
sock := flag.String("socket", "/tmp/zhiyi-ipc.sock", "rust sidecar unix socket")
|
||
apply := flag.Bool("apply", false, "真正写库(默认 false = dry-run;未经终审禁止 true)")
|
||
flag.Parse()
|
||
|
||
resp, err := rpc(*sock, ipcReq{Type: "lancedb_scan_all"})
|
||
if err != nil {
|
||
fmt.Fprintln(os.Stderr, "scan_all 失败:", err)
|
||
os.Exit(1)
|
||
}
|
||
var rows []row
|
||
if err := json.Unmarshal([]byte(resp.ReportJSON), &rows); err != nil {
|
||
fmt.Fprintln(os.Stderr, "unmarshal 失败:", err)
|
||
os.Exit(1)
|
||
}
|
||
|
||
var nu []row
|
||
for _, r := range rows {
|
||
if r.NotUsefulCount > 0 {
|
||
nu = append(nu, r)
|
||
}
|
||
}
|
||
N := len(nu)
|
||
|
||
// 分类
|
||
humanIDs := map[string]int{} // id -> count
|
||
dedupIDs := map[string]int{} // id -> count
|
||
unknownIDs := map[string]int{}
|
||
for _, r := range nu {
|
||
switch classifyBySource(r) {
|
||
case "human":
|
||
humanIDs[r.ID] = r.NotUsefulCount
|
||
case "dedup":
|
||
dedupIDs[r.ID] = r.NotUsefulCount
|
||
default:
|
||
unknownIDs[r.ID] = r.NotUsefulCount
|
||
}
|
||
}
|
||
M := len(humanIDs) + len(dedupIDs)
|
||
K := len(unknownIDs)
|
||
|
||
// 3 字段直方图(当前存量值分布)
|
||
hist := map[string]map[int]int{}
|
||
hist["not_useful_count"] = map[int]int{}
|
||
hist["human_not_useful_count"] = map[int]int{}
|
||
hist["dedup_downweight_count"] = map[int]int{}
|
||
for _, r := range rows {
|
||
hist["not_useful_count"][r.NotUsefulCount]++
|
||
hist["human_not_useful_count"][r.HumanNotUsefulCount]++
|
||
hist["dedup_downweight_count"][r.DedupDownweightCount]++
|
||
}
|
||
|
||
fmt.Printf("总行数(存活): %d\n", len(rows))
|
||
fmt.Printf("not_useful_count>0 待改行数 N = %d\n", N)
|
||
fmt.Printf(" 其中可按来源判定 M = %d (human=%d dedup=%d)\n", M, len(humanIDs), len(dedupIDs))
|
||
fmt.Printf(" 无法判定归 unknown K = %d\n", K)
|
||
fmt.Println("── 3 字段直方图(value→行数,仅列当前非零分布)──")
|
||
order := []string{"not_useful_count", "human_not_useful_count", "dedup_downweight_count"}
|
||
for _, f := range order {
|
||
m := hist[f]
|
||
keys := make([]int, 0, len(m))
|
||
for k := range m {
|
||
keys = append(keys, k)
|
||
}
|
||
sort.Ints(keys)
|
||
fmt.Printf("[%s] ", f)
|
||
if len(keys) == 1 && keys[0] == 0 {
|
||
fmt.Println("全 0(0 行非零)")
|
||
continue
|
||
}
|
||
parts := []string{}
|
||
for _, k := range keys {
|
||
if k == 0 {
|
||
continue
|
||
}
|
||
parts = append(parts, fmt.Sprintf("%d×%d", k, m[k]))
|
||
}
|
||
fmt.Println(parts)
|
||
}
|
||
|
||
if *apply {
|
||
if M == 0 {
|
||
fmt.Println("apply: 无可判定行(M=0),无需迁移,存量 not_useful_count 保持原样。")
|
||
} else {
|
||
fmt.Printf("apply: 将拆分 %d 行(human=%d dedup=%d)—— 需逐条 Update。\n", M, len(humanIDs), len(dedupIDs))
|
||
}
|
||
} else {
|
||
fmt.Println("(dry-run) 未写库。输出须先经小怡终审,批准后才可 --apply。")
|
||
}
|
||
}
|