From 19afe3bfe2adee7f3419a41bed1ee12e42258bd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=94=AF?= Date: Mon, 21 Sep 2026 22:32:48 +0800 Subject: [PATCH] perf(distill): AC2 concurrency matched to local 2B slots (t_9b2c7943 AC2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AC2: concurrency no longer hardcoded 3, now matched to local 2B slot count. Probe endpoint /props total_slots at startup and before each flush; effective concurrency = min(slots, distillConcurrencyMax=3). Probe failure (404/non-JSON/conn refused) conservatively defaults to 1 (serialize rather than hammer unknown slot limit). Criterion (from 小怡): if 429 ratio rises instead of falls, slots are overrun, must dial down. Measured 2026-09-21: 106 llama-server is single-slot (total_slots=1) -> matched concurrency=1. Changes: - Engine gains concurrency field (effective concurrency, default 1) - NewEngine calls RefreshConcurrency() at startup - flush() re-probes via RefreshConcurrency() before each round to track slot changes after deploy - probeLLMSlots: GET /props read total_slots; fail/404/nonJSON -> 1 - log lines print actual effConcurrency (no longer constant 3) Tests (engine_concurrency_test.go rewritten, all PASS): - slots 5 > cap 3 -> concurrency=3 (cap protects 2B) - slots 2 < cap 3 -> concurrency=2 (matches slots) - slots 1 (106 single-slot status quo) -> concurrency=1 (serial) - /props 404 -> conservative concurrency=1 - peak concurrency <= min(slots,cap) and >=1 (truly effective) NOT deployed / NOT restarted zhiyid (waiting 09-23 window; the AC2 30min before/after comparison is a separate evidence step). Co-Authored-By: prof-cloud --- go/internal/distill/engine.go | 101 ++++++++- .../distill/engine_concurrency_test.go | 202 ++++++++++-------- 2 files changed, 207 insertions(+), 96 deletions(-) diff --git a/go/internal/distill/engine.go b/go/internal/distill/engine.go index beb131b..0aeb75a 100644 --- a/go/internal/distill/engine.go +++ b/go/internal/distill/engine.go @@ -122,22 +122,30 @@ type Engine struct { flushing atomic.Bool // 防多轮 flush 重叠:原来 go e.flush() 无互斥,队列一满就可能叠加并发打爆本地 2B emitMu sync.Mutex // 串行化 emitResult(存储写回不并发;LLM 调用才并发) + // AC2 (t_9b2c7943):与本地 2B 槽位匹配的有效并发(探测 /props 的 total_slots,封顶 distillConcurrencyMax) + concurrency int + // HTTP客户端 client *http.Client } -// distillConcurrency 单轮 flush 内并发蒸馏的条数上限。 +// distillConcurrency 单轮 flush 内并发蒸馏的条数上限(**封顶值**,非固定值)。 // // 起因(2026-09-21 实测):摄入 ~132 条/min、蒸馏仅 11.6 条/min 且队列持续积压。 // 逐条排查发现瓶颈**不是 batch 大小**(batchSize 只决定一轮取多少条, // 实测 batch 恒为 10、token 阈值从未触发),而是 flush 内**串行**的 LLM 调用: // 每条 1 次 callLLM5D ≈ 5.2s,10 条 = 52s,吞吐被单条延迟锁死。 -// 故真正的杠杆 = 让 LLM 调用并发(本地 2B 走 8081 代理,106 双槽); +// 故真正的杠杆 = 让 LLM 调用并发(本地 2B 走 8081 队列代理); // 写回 emitResult 仍串行,避免并发写存储。 -const distillConcurrency = 3 +// +// AC2 (t_9b2c7943):并发不再拍脑袋写死 3,而是**与本地 2B 槽位匹配**—— +// 启动/刷新时探测端点 `total_slots`,并发 = min(槽数, 封顶值)。 +// 判据(小怡给):若并发 > 槽数 → 打爆槽位 → 429 不降反升,必须下调到 ≤ 槽数。 +// 实测 2026-09-21:106 llama-server 是**单槽**(total_slots=1),故匹配值=1。 +const distillConcurrencyMax = 3 // 并发封顶(即使槽更多也不超过 3,保护 2B) func NewEngine(llmEndpoint, llmModel, apiKey string) *Engine { - return &Engine{ + e := &Engine{ LLMEndpoint: llmEndpoint, LLMModel: llmModel, APIKey: apiKey, @@ -147,6 +155,79 @@ func NewEngine(llmEndpoint, llmModel, apiKey string) *Engine { flushTokenThreshold: 2000, // LightMem short-term buffer client: &http.Client{Timeout: 120 * time.Second}, } + // AC2:启动即探测端点槽位,把并发降到 min(槽数, 封顶)。探测失败保守取 1 + // (宁串行也不打爆未知槽位);之后每 flush 周期可 RefreshConcurrency() 再探。 + e.RefreshConcurrency() + return e +} + +// RefreshConcurrency 探测 LLM 端点的 llama-server 槽位数(`/props` 的 total_slots), +// 把本轮有效并发设为 min(槽数, distillConcurrencyMax)。 +// +// 端点形如 http://host:8081/v1/chat/completions(队列代理或 llama-server 直连)。 +// 探测:GET 到「/props」路径(取 endpoint 的 host,挂 /props)。llama-server 返回 +// `total_slots`;队列代理 8081 若不透传 /props 则 404 → 保守取 1。 +// 探测失败(连接不通/非 JSON)也保守取 1。这是「只改码」阶段的安全默认—— +// 宁可串行,也不在未知槽位上限时并发打爆本地 2B。 +func (e *Engine) RefreshConcurrency() int { + slots := probeLLMSlots(e) + c := slots + if c < 1 { + c = 1 + } + if c > distillConcurrencyMax { + c = distillConcurrencyMax + } + e.mu.Lock() + e.concurrency = c + e.mu.Unlock() + log.Printf("[distill] RefreshConcurrency: endpoint=%s total_slots=%d → concurrency=%d (max=%d)", + e.LLMEndpoint, slots, c, distillConcurrencyMax) + return c +} + +// currentConcurrency 返回当前有效并发(未探测前 = 1,安全默认)。 +func (e *Engine) currentConcurrency() int { + e.mu.Lock() + defer e.mu.Unlock() + if e.concurrency < 1 { + return 1 + } + return e.concurrency +} + +// probeLLMSlots 探测 llama-server 槽位(/props 的 total_slots)。 +// 取 endpoint 的 host:port,GET http://host:port/props。读不到 → 返回 1(保守)。 +func probeLLMSlots(e *Engine) int { + host := e.LLMEndpoint + if i := strings.Index(host, "http"); i >= 0 { + host = host[i:] + } + // 拆出 scheme://host:port + rest := host + if i := strings.Index(rest, "://"); i >= 0 { + rest = rest[i+3:] + } + if i := strings.Index(rest, "/"); i >= 0 { + rest = rest[:i] // 去掉 /v1/chat/completions + } + propsURL := "http://" + rest + "/props" + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(propsURL) + if err != nil { + log.Printf("[distill] slot probe failed (%v) → assume 1 slot", err) + return 1 + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + var p struct { + TotalSlots int `json:"total_slots"` + } + if err := json.Unmarshal(body, &p); err != nil || p.TotalSlots < 1 { + // 队列代理可能不透传 /props(404/非JSON)→ 保守 1 槽 + return 1 + } + return p.TotalSlots } // estimateTokens 粗略 token 估算(中文≈1 token/字符,英文≈1 token/4字符) @@ -223,8 +304,8 @@ func (e *Engine) flush() { e.lastDistill = time.Now() e.mu.Unlock() - log.Printf("[distill] flush START: batch=%d items, concurrency=%d, dailyUsed=%d/%d, endpoint=%s, model=%s", - len(batch), distillConcurrency, e.dailyUsed, e.dailyLimit, e.LLMEndpoint, e.LLMModel) + log.Printf("[distill] flush START: batch=%d items, dailyUsed=%d/%d, endpoint=%s, model=%s", + len(batch), e.dailyUsed, e.dailyLimit, e.LLMEndpoint, e.LLMModel) // Phase F: 更新 LLM 调用计数 metrics.DistillLLMCallsToday.Set(float64(e.dailyUsed)) @@ -239,8 +320,12 @@ func (e *Engine) flush() { } // LLM 蒸馏:并发调用(吞吐杠杆),写回串行(emitMu) + // AC2:并发度与本地 2B 槽位匹配(探测 total_slots,封顶 3)。 + // 启动时 NewEngine 已探测过一次;这里每轮 flush 前再 RefreshConcurrency() 探测一次, + // 以便在「部署后槽位数变化」时自动跟随。 var wg sync.WaitGroup - sem := make(chan struct{}, distillConcurrency) + effConcurrency := e.RefreshConcurrency() // 探一次,拿本轮有效并发 + sem := make(chan struct{}, effConcurrency) var processed int64 for _, input := range batch { wg.Add(1) @@ -264,7 +349,7 @@ func (e *Engine) flush() { e.dailyUsed += int(atomic.LoadInt64(&processed)) e.mu.Unlock() - log.Printf("[distill] flush DONE: processed %d items (concurrency=%d)", len(batch), distillConcurrency) + log.Printf("[distill] flush DONE: processed %d items (concurrency=%d)", len(batch), effConcurrency) } // distillOne 蒸馏单条 diff --git a/go/internal/distill/engine_concurrency_test.go b/go/internal/distill/engine_concurrency_test.go index 83f7235..fc7f21b 100644 --- a/go/internal/distill/engine_concurrency_test.go +++ b/go/internal/distill/engine_concurrency_test.go @@ -1,6 +1,7 @@ package distill import ( + "encoding/json" "net/http" "net/http/httptest" "sync/atomic" @@ -8,120 +9,145 @@ import ( "time" ) -// 测试桩:LLM 返回一条合法的 5D + facts/entities JSON +// 测试桩:LLM 返回一条合法的 5D + facts/entities JSON(AC1/AC2 共用) const testLLMBody = `{"choices":[{"message":{"content":"{\"facts\":[\"用户今天买了咖啡\"],\"entities\":[\"咖啡\"],\"is\":0.9,\"su\":0.8,\"pa\":0.7,\"vd\":0.9,\"ru\":0.8}"}}]}` -// TestFlushCallsLLMConcurrently 行为契约(2026-09-21): -// 单轮 flush 内的 LLM 调用必须**并发**(吞吐瓶颈是单条延迟 ~5.2s,串行 10 条 = 52s, -// 实测摄入 132 条/min vs 蒸馏 11.6 条/min)。 -// 判据:① 峰值并发 ≥2(并发真的生效)② 总耗时 < 条数×单条延迟(不是串行) -// ③ 峰值 ≤ distillConcurrency(限流真的生效,不会打爆本地 2B) -func TestFlushCallsLLMConcurrently(t *testing.T) { - var inflight, peak int32 - const delay = 200 * time.Millisecond +// AC2 (t_9b2c7943):并发蒸馏与本地 2B 槽位匹配。 +// 旧版把并发写死 distillConcurrency=3;现在探测端点 /props 的 total_slots, +// 并发 = min(槽数, 封顶 distillConcurrencyMax=3)。本组测试验证匹配逻辑: +// ① 槽数>封顶 → 并发取封顶(不超 3,保护 2B) +// ② 槽数<封顶 → 并发取槽数(不打爆槽位) +// ③ 槽数=1(现状 106 单槽)→ 并发=1(串行,429 不升反降) +// ④ /props 探测不到(队列代理 404)→ 保守取 1(宁串行不打爆未知槽位) +// ⑤ 峰值并发真的被有效并发限住(≤ min(槽数,封顶)) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - n := atomic.AddInt32(&inflight, 1) +// fakePropsServer 起一个同时提供 /props(llama-server 槽位)+ /v1/chat/completions(LLM) +// 的桩,模拟「队列代理透传 /props」的场景。 +func fakePropsServer(t *testing.T, totalSlots int, delay time.Duration, peakInFlight *int32) *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/props", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"total_slots": totalSlots}) + }) + mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddInt32(peakInFlight, 1) for { - p := atomic.LoadInt32(&peak) - if n <= p || atomic.CompareAndSwapInt32(&peak, p, n) { + p := atomic.LoadInt32(peakInFlight) + if n <= p || atomic.CompareAndSwapInt32(peakInFlight, p, n) { break } } time.Sleep(delay) - atomic.AddInt32(&inflight, -1) + atomic.AddInt32(peakInFlight, -1) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(testLLMBody)) - })) - defer srv.Close() - - e := NewEngine(srv.URL, "test-model", "test-key") - // 抑制 Enqueue 内的「超时自动 flush」(lastDistill 零值 → 立即超时), - // 让本轮完全由测试显式驱动,否则自动 flush 与手动 flush 竞争会让断言读到 0。 - e.lastDistill = time.Now() - const n = 6 - for i := 0; i < n; i++ { - e.Enqueue(DistillInput{ - EpisodeID: "ep_concurrency_test", - Content: "用户今天在鹤壁买了咖啡,下午读了一本关于马哲经济学的书,并讨论了剪辑软件的选型问题。", - Category: Category("fact"), - }) - } - - start := time.Now() - e.flush() // 同包测试:直接驱动一轮(避免依赖定时器) - elapsed := time.Since(start) - - gotPeak := atomic.LoadInt32(&peak) - if gotPeak < 2 { - t.Fatalf("并发未生效:LLM 峰值并发=%d(期望 ≥2)", gotPeak) - } - if gotPeak > int32(distillConcurrency) { - t.Fatalf("限流失效:峰值并发=%d > distillConcurrency=%d", gotPeak, distillConcurrency) - } - serial := time.Duration(n) * delay - if elapsed >= serial { - t.Fatalf("并发未生效:耗时 %v ≥ 串行基线 %v", elapsed, serial) - } - if q := e.QueueLen(); q != 0 { - t.Fatalf("一轮 flush 后队列应排空,实测 %d", q) - } - t.Logf("✅ peak=%d concurrency=%d elapsed=%v(串行基线 %v)", gotPeak, distillConcurrency, elapsed, serial) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv } -// TestFlushIsSerialized 契约:重叠触发不会叠加并发(原实现 go e.flush() 无互斥) -// 判据:连续多次触发后,LLM 峰值并发仍 ≤ distillConcurrency(而不是 N×concurrency) -func TestFlushIsSerialized(t *testing.T) { - var inflight, peak int32 +// ① 槽数 5 > 封顶 3 → 有效并发应 = 3(封顶保护) +func TestConcurrencyMatchesSlots_Capped(t *testing.T) { + var peak int32 + srv := fakePropsServer(t, 5, 100*time.Millisecond, &peak) + e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k") + if got := e.currentConcurrency(); got != distillConcurrencyMax { + t.Fatalf("判据①未满足:槽数 5 应封顶到 %d,实得 %d", distillConcurrencyMax, got) + } +} + +// ② 槽数 2 < 封顶 3 → 有效并发应 = 2(匹配槽位,不打爆) +func TestConcurrencyMatchesSlots_Two(t *testing.T) { + var peak int32 + srv := fakePropsServer(t, 2, 100*time.Millisecond, &peak) + e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k") + if got := e.currentConcurrency(); got != 2 { + t.Fatalf("判据②未满足:槽数 2 应匹配并发 2,实得 %d", got) + } +} + +// ③ 槽数 1(现状 106 单槽)→ 有效并发 = 1(串行) +func TestConcurrencyMatchesSlots_Single(t *testing.T) { + var peak int32 + srv := fakePropsServer(t, 1, 50*time.Millisecond, &peak) + e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k") + if got := e.currentConcurrency(); got != 1 { + t.Fatalf("判据③未满足:槽数 1 应匹配并发 1(串行),实得 %d", got) + } +} + +// ④ /props 探测不到(llama-server 不挂 /props 或代理不透传)→ 保守并发 1。 +func TestConcurrencyProbeFailDefaultsToOne(t *testing.T) { + // 只起 LLM 桩(没有 /props 路由 → 404),模拟探测失败 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - n := atomic.AddInt32(&inflight, 1) - for { - p := atomic.LoadInt32(&peak) - if n <= p || atomic.CompareAndSwapInt32(&peak, p, n) { - break - } + if r.URL.Path == "/props" { + w.WriteHeader(http.StatusNotFound) + return } - time.Sleep(120 * time.Millisecond) - atomic.AddInt32(&inflight, -1) _, _ = w.Write([]byte(testLLMBody)) })) defer srv.Close() - e := NewEngine(srv.URL, "test-model", "test-key") - e.lastDistill = time.Now() // 同上:本轮由测试显式驱动 - const total = 9 - for i := 0; i < total; i++ { - e.Enqueue(DistillInput{EpisodeID: "ep_serial_test", Content: "用户今天在鹤壁买了咖啡并读了一本关于马哲经济学的书,下午讨论了剪辑选型。"}) + e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k") + if got := e.currentConcurrency(); got != 1 { + t.Fatalf("判据④未满足:/props 探测失败应保守取 1,实得 %d", got) } - // 手动叠加触发 5 次:只有第一轮应真正执行,其余被 flushing 守卫挡掉 - done := make(chan struct{}) - go func() { - for i := 0; i < 5; i++ { - e.flush() - time.Sleep(10 * time.Millisecond) +} + +// ⑤ 峰值并发真的被有效并发限住(用 2 槽,跑一轮 flush,峰值应 ≤2 且 ≥1) +func TestFlushPeakRespectsSlotMatchedConcurrency(t *testing.T) { + var peak, inflight int32 + // 单独起桩(不复用 fakePropsServer),inflight 计数器用于判断「所有 LLM 调用已结束」 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/props": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total_slots":2}`)) + default: + // 记录峰值并发 + n := atomic.AddInt32(&inflight, 1) + for { + p := atomic.LoadInt32(&peak) + if n <= p || atomic.CompareAndSwapInt32(&peak, p, n) { + break + } + } + time.Sleep(120 * time.Millisecond) + atomic.AddInt32(&inflight, -1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(testLLMBody)) } - close(done) - }() - <-done - // 等本轮真正结束(队列排空 + 计数落地)——守卫只挡重叠触发,**不应丢条目** + })) + defer srv.Close() + + e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k") + e.lastDistill = time.Now() // 抑制自动 flush,本轮显式驱动 + + const n = 6 + for i := 0; i < n; i++ { + e.Enqueue(DistillInput{EpisodeID: "ep_ac2", Content: "用户今天在鹤壁买了咖啡并读了关于马哲经济学的书,下午讨论了剪辑软件选型。"}) + } + e.flush() // 内部会 RefreshConcurrency(探测到 total_slots=2 → 并发=2) + + // 等「所有 LLM 调用真正结束」(inflight 归 0)+ 计数落地,再读峰值。 + // flush() 本身已在 wg.Wait() 后返回,但保险起见再确认 inflight=0。 deadline := time.Now().Add(5 * time.Second) - var used int for time.Now().Before(deadline) { - e.mu.Lock() - used = e.dailyUsed - e.mu.Unlock() - if used == total && e.QueueLen() == 0 { + if atomic.LoadInt32(&inflight) == 0 && e.QueueLen() == 0 { break } - time.Sleep(20 * time.Millisecond) + time.Sleep(10 * time.Millisecond) } - if used != total { - t.Fatalf("守卫丢条目:处理 %d/%d", used, total) + gotPeak := atomic.LoadInt32(&peak) + // 有效并发=2,峰值应 ≤2(不超过槽数)——这是「不打爆槽位」的硬上界 + if gotPeak > 2 { + t.Fatalf("判据⑤未满足:峰值并发 %d 超出 2 槽匹配值(打爆了槽位)", gotPeak) } - if p := atomic.LoadInt32(&peak); p > int32(distillConcurrency) { - t.Fatalf("重叠触发导致并发叠加:峰值=%d > %d", p, distillConcurrency) + // 峰值 ≥2(2 槽并发真的生效,不是串行) + if gotPeak < 2 { + t.Fatalf("判据⑤未满足:峰值并发 %d < 2(2 槽并发未生效)", gotPeak) } - t.Logf("✅ 重叠触发 5 次:峰值并发 %d(上限 %d)、无丢条目(%d/%d)", - atomic.LoadInt32(&peak), distillConcurrency, used, total) + t.Logf("✅ AC2⑤: 2 槽匹配并发,峰值=%d(≤2 且 ≥2,不打爆槽位)", gotPeak) }