perf(distill): AC2 concurrency matched to local 2B slots (t_9b2c7943 AC2)

AC2: concurrency no longer hardcoded 3, now matched to local 2B slot count.
Probe endpoint /props total_slots at startup and before each flush;
effective concurrency = min(slots, distillConcurrencyMax=3).
Probe failure (404/non-JSON/conn refused) conservatively defaults to 1
(serialize rather than hammer unknown slot limit).

Criterion (from 小怡): if 429 ratio rises instead of falls, slots are
overrun, must dial down. Measured 2026-09-21: 106 llama-server is
single-slot (total_slots=1) -> matched concurrency=1.

Changes:
- Engine gains concurrency field (effective concurrency, default 1)
- NewEngine calls RefreshConcurrency() at startup
- flush() re-probes via RefreshConcurrency() before each round to
  track slot changes after deploy
- probeLLMSlots: GET /props read total_slots; fail/404/nonJSON -> 1
- log lines print actual effConcurrency (no longer constant 3)

Tests (engine_concurrency_test.go rewritten, all PASS):
- slots 5 > cap 3 -> concurrency=3 (cap protects 2B)
- slots 2 < cap 3 -> concurrency=2 (matches slots)
- slots 1 (106 single-slot status quo) -> concurrency=1 (serial)
- /props 404 -> conservative concurrency=1
- peak concurrency <= min(slots,cap) and >=1 (truly effective)

NOT deployed / NOT restarted zhiyid (waiting 09-23 window; the AC2
30min before/after comparison is a separate evidence step).

Co-Authored-By: prof-cloud <prof-cloud@hermes>
This commit is contained in:
小唯
2026-09-21 22:32:48 +08:00
co-authored by prof-cloud
parent 4f947cecef
commit 19afe3bfe2
2 changed files with 207 additions and 96 deletions
+93 -8
View File
@@ -122,22 +122,30 @@ type Engine struct {
flushing atomic.Bool // 防多轮 flush 重叠:原来 go e.flush() 无互斥,队列一满就可能叠加并发打爆本地 2B
emitMu sync.Mutex // 串行化 emitResult(存储写回不并发;LLM 调用才并发)
// AC2 (t_9b2c7943):与本地 2B 槽位匹配的有效并发(探测 /props 的 total_slots,封顶 distillConcurrencyMax)
concurrency int
// HTTP客户端
client *http.Client
}
// distillConcurrency 单轮 flush 内并发蒸馏的条数上限。
// distillConcurrency 单轮 flush 内并发蒸馏的条数上限(**封顶值**,非固定值)。
//
// 起因(2026-09-21 实测):摄入 ~132 条/min、蒸馏仅 11.6 条/min 且队列持续积压。
// 逐条排查发现瓶颈**不是 batch 大小**(batchSize 只决定一轮取多少条,
// 实测 batch 恒为 10、token 阈值从未触发),而是 flush 内**串行**的 LLM 调用:
// 每条 1 次 callLLM5D ≈ 5.2s,10 条 = 52s,吞吐被单条延迟锁死。
// 故真正的杠杆 = 让 LLM 调用并发(本地 2B 走 8081 代理,106 双槽);
// 故真正的杠杆 = 让 LLM 调用并发(本地 2B 走 8081 队列代理);
// 写回 emitResult 仍串行,避免并发写存储。
const distillConcurrency = 3
//
// AC2 (t_9b2c7943):并发不再拍脑袋写死 3,而是**与本地 2B 槽位匹配**——
// 启动/刷新时探测端点 `total_slots`,并发 = min(槽数, 封顶值)。
// 判据(小怡给):若并发 > 槽数 → 打爆槽位 → 429 不降反升,必须下调到 ≤ 槽数。
// 实测 2026-09-21:106 llama-server 是**单槽**(total_slots=1),故匹配值=1。
const distillConcurrencyMax = 3 // 并发封顶(即使槽更多也不超过 3,保护 2B)
func NewEngine(llmEndpoint, llmModel, apiKey string) *Engine {
return &Engine{
e := &Engine{
LLMEndpoint: llmEndpoint,
LLMModel: llmModel,
APIKey: apiKey,
@@ -147,6 +155,79 @@ func NewEngine(llmEndpoint, llmModel, apiKey string) *Engine {
flushTokenThreshold: 2000, // LightMem short-term buffer
client: &http.Client{Timeout: 120 * time.Second},
}
// AC2:启动即探测端点槽位,把并发降到 min(槽数, 封顶)。探测失败保守取 1
// (宁串行也不打爆未知槽位);之后每 flush 周期可 RefreshConcurrency() 再探。
e.RefreshConcurrency()
return e
}
// RefreshConcurrency 探测 LLM 端点的 llama-server 槽位数(`/props` 的 total_slots),
// 把本轮有效并发设为 min(槽数, distillConcurrencyMax)。
//
// 端点形如 http://host:8081/v1/chat/completions(队列代理或 llama-server 直连)。
// 探测:GET 到「/props」路径(取 endpoint 的 host,挂 /props)。llama-server 返回
// `total_slots`;队列代理 8081 若不透传 /props 则 404 → 保守取 1。
// 探测失败(连接不通/非 JSON)也保守取 1。这是「只改码」阶段的安全默认——
// 宁可串行,也不在未知槽位上限时并发打爆本地 2B。
func (e *Engine) RefreshConcurrency() int {
slots := probeLLMSlots(e)
c := slots
if c < 1 {
c = 1
}
if c > distillConcurrencyMax {
c = distillConcurrencyMax
}
e.mu.Lock()
e.concurrency = c
e.mu.Unlock()
log.Printf("[distill] RefreshConcurrency: endpoint=%s total_slots=%d → concurrency=%d (max=%d)",
e.LLMEndpoint, slots, c, distillConcurrencyMax)
return c
}
// currentConcurrency 返回当前有效并发(未探测前 = 1,安全默认)。
func (e *Engine) currentConcurrency() int {
e.mu.Lock()
defer e.mu.Unlock()
if e.concurrency < 1 {
return 1
}
return e.concurrency
}
// probeLLMSlots 探测 llama-server 槽位(/props 的 total_slots)。
// 取 endpoint 的 host:port,GET http://host:port/props。读不到 → 返回 1(保守)。
func probeLLMSlots(e *Engine) int {
host := e.LLMEndpoint
if i := strings.Index(host, "http"); i >= 0 {
host = host[i:]
}
// 拆出 scheme://host:port
rest := host
if i := strings.Index(rest, "://"); i >= 0 {
rest = rest[i+3:]
}
if i := strings.Index(rest, "/"); i >= 0 {
rest = rest[:i] // 去掉 /v1/chat/completions
}
propsURL := "http://" + rest + "/props"
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Get(propsURL)
if err != nil {
log.Printf("[distill] slot probe failed (%v) → assume 1 slot", err)
return 1
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
var p struct {
TotalSlots int `json:"total_slots"`
}
if err := json.Unmarshal(body, &p); err != nil || p.TotalSlots < 1 {
// 队列代理可能不透传 /props(404/非JSON)→ 保守 1 槽
return 1
}
return p.TotalSlots
}
// estimateTokens 粗略 token 估算(中文≈1 token/字符,英文≈1 token/4字符)
@@ -223,8 +304,8 @@ func (e *Engine) flush() {
e.lastDistill = time.Now()
e.mu.Unlock()
log.Printf("[distill] flush START: batch=%d items, concurrency=%d, dailyUsed=%d/%d, endpoint=%s, model=%s",
len(batch), distillConcurrency, e.dailyUsed, e.dailyLimit, e.LLMEndpoint, e.LLMModel)
log.Printf("[distill] flush START: batch=%d items, dailyUsed=%d/%d, endpoint=%s, model=%s",
len(batch), e.dailyUsed, e.dailyLimit, e.LLMEndpoint, e.LLMModel)
// Phase F: 更新 LLM 调用计数
metrics.DistillLLMCallsToday.Set(float64(e.dailyUsed))
@@ -239,8 +320,12 @@ func (e *Engine) flush() {
}
// LLM 蒸馏:并发调用(吞吐杠杆),写回串行(emitMu)
// AC2:并发度与本地 2B 槽位匹配(探测 total_slots,封顶 3)。
// 启动时 NewEngine 已探测过一次;这里每轮 flush 前再 RefreshConcurrency() 探测一次,
// 以便在「部署后槽位数变化」时自动跟随。
var wg sync.WaitGroup
sem := make(chan struct{}, distillConcurrency)
effConcurrency := e.RefreshConcurrency() // 探一次,拿本轮有效并发
sem := make(chan struct{}, effConcurrency)
var processed int64
for _, input := range batch {
wg.Add(1)
@@ -264,7 +349,7 @@ func (e *Engine) flush() {
e.dailyUsed += int(atomic.LoadInt64(&processed))
e.mu.Unlock()
log.Printf("[distill] flush DONE: processed %d items (concurrency=%d)", len(batch), distillConcurrency)
log.Printf("[distill] flush DONE: processed %d items (concurrency=%d)", len(batch), effConcurrency)
}
// distillOne 蒸馏单条
+114 -88
View File
@@ -1,6 +1,7 @@
package distill
import (
"encoding/json"
"net/http"
"net/http/httptest"
"sync/atomic"
@@ -8,120 +9,145 @@ import (
"time"
)
// 测试桩:LLM 返回一条合法的 5D + facts/entities JSON
// 测试桩:LLM 返回一条合法的 5D + facts/entities JSON(AC1/AC2 共用)
const testLLMBody = `{"choices":[{"message":{"content":"{\"facts\":[\"用户今天买了咖啡\"],\"entities\":[\"咖啡\"],\"is\":0.9,\"su\":0.8,\"pa\":0.7,\"vd\":0.9,\"ru\":0.8}"}}]}`
// TestFlushCallsLLMConcurrently 行为契约(2026-09-21):
// 单轮 flush 内的 LLM 调用必须**并发**(吞吐瓶颈是单条延迟 ~5.2s,串行 10 条 = 52s,
// 实测摄入 132 条/min vs 蒸馏 11.6 条/min)。
// 判据:① 峰值并发 ≥2(并发真的生效)② 总耗时 < 条数×单条延迟(不是串行)
// ③ 峰值 ≤ distillConcurrency(限流真的生效,不会打爆本地 2B)
func TestFlushCallsLLMConcurrently(t *testing.T) {
var inflight, peak int32
const delay = 200 * time.Millisecond
// AC2 (t_9b2c7943):并发蒸馏与本地 2B 槽位匹配。
// 旧版把并发写死 distillConcurrency=3;现在探测端点 /props 的 total_slots,
// 并发 = min(槽数, 封顶 distillConcurrencyMax=3)。本组测试验证匹配逻辑:
// ① 槽数>封顶 → 并发取封顶(不超 3,保护 2B)
// ② 槽数<封顶 → 并发取槽数(不打爆槽位)
// ③ 槽数=1(现状 106 单槽)→ 并发=1(串行,429 不升反降)
// ④ /props 探测不到(队列代理 404)→ 保守取 1(宁串行不打爆未知槽位)
// ⑤ 峰值并发真的被有效并发限住(≤ min(槽数,封顶))
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt32(&inflight, 1)
// fakePropsServer 起一个同时提供 /props(llama-server 槽位)+ /v1/chat/completions(LLM)
// 的桩,模拟「队列代理透传 /props」的场景。
func fakePropsServer(t *testing.T, totalSlots int, delay time.Duration, peakInFlight *int32) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/props", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]interface{}{"total_slots": totalSlots})
})
mux.HandleFunc("/v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt32(peakInFlight, 1)
for {
p := atomic.LoadInt32(&peak)
if n <= p || atomic.CompareAndSwapInt32(&peak, p, n) {
p := atomic.LoadInt32(peakInFlight)
if n <= p || atomic.CompareAndSwapInt32(peakInFlight, p, n) {
break
}
}
time.Sleep(delay)
atomic.AddInt32(&inflight, -1)
atomic.AddInt32(peakInFlight, -1)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(testLLMBody))
}))
defer srv.Close()
e := NewEngine(srv.URL, "test-model", "test-key")
// 抑制 Enqueue 内的「超时自动 flush」(lastDistill 零值 → 立即超时),
// 让本轮完全由测试显式驱动,否则自动 flush 与手动 flush 竞争会让断言读到 0。
e.lastDistill = time.Now()
const n = 6
for i := 0; i < n; i++ {
e.Enqueue(DistillInput{
EpisodeID: "ep_concurrency_test",
Content: "用户今天在鹤壁买了咖啡,下午读了一本关于马哲经济学的书,并讨论了剪辑软件的选型问题。",
Category: Category("fact"),
})
}
start := time.Now()
e.flush() // 同包测试:直接驱动一轮(避免依赖定时器)
elapsed := time.Since(start)
gotPeak := atomic.LoadInt32(&peak)
if gotPeak < 2 {
t.Fatalf("并发未生效:LLM 峰值并发=%d(期望 ≥2)", gotPeak)
}
if gotPeak > int32(distillConcurrency) {
t.Fatalf("限流失效:峰值并发=%d > distillConcurrency=%d", gotPeak, distillConcurrency)
}
serial := time.Duration(n) * delay
if elapsed >= serial {
t.Fatalf("并发未生效:耗时 %v ≥ 串行基线 %v", elapsed, serial)
}
if q := e.QueueLen(); q != 0 {
t.Fatalf("一轮 flush 后队列应排空,实测 %d", q)
}
t.Logf("✅ peak=%d concurrency=%d elapsed=%v(串行基线 %v)", gotPeak, distillConcurrency, elapsed, serial)
})
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
// TestFlushIsSerialized 契约:重叠触发不会叠加并发(原实现 go e.flush() 无互斥)
// 判据:连续多次触发后,LLM 峰值并发仍 ≤ distillConcurrency(而不是 N×concurrency)
func TestFlushIsSerialized(t *testing.T) {
var inflight, peak int32
// ① 槽数 5 > 封顶 3 → 有效并发应 = 3(封顶保护)
func TestConcurrencyMatchesSlots_Capped(t *testing.T) {
var peak int32
srv := fakePropsServer(t, 5, 100*time.Millisecond, &peak)
e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k")
if got := e.currentConcurrency(); got != distillConcurrencyMax {
t.Fatalf("判据①未满足:槽数 5 应封顶到 %d,实得 %d", distillConcurrencyMax, got)
}
}
// ② 槽数 2 < 封顶 3 → 有效并发应 = 2(匹配槽位,不打爆)
func TestConcurrencyMatchesSlots_Two(t *testing.T) {
var peak int32
srv := fakePropsServer(t, 2, 100*time.Millisecond, &peak)
e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k")
if got := e.currentConcurrency(); got != 2 {
t.Fatalf("判据②未满足:槽数 2 应匹配并发 2,实得 %d", got)
}
}
// ③ 槽数 1(现状 106 单槽)→ 有效并发 = 1(串行)
func TestConcurrencyMatchesSlots_Single(t *testing.T) {
var peak int32
srv := fakePropsServer(t, 1, 50*time.Millisecond, &peak)
e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k")
if got := e.currentConcurrency(); got != 1 {
t.Fatalf("判据③未满足:槽数 1 应匹配并发 1(串行),实得 %d", got)
}
}
// ④ /props 探测不到(llama-server 不挂 /props 或代理不透传)→ 保守并发 1。
func TestConcurrencyProbeFailDefaultsToOne(t *testing.T) {
// 只起 LLM 桩(没有 /props 路由 → 404),模拟探测失败
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := atomic.AddInt32(&inflight, 1)
for {
p := atomic.LoadInt32(&peak)
if n <= p || atomic.CompareAndSwapInt32(&peak, p, n) {
break
}
if r.URL.Path == "/props" {
w.WriteHeader(http.StatusNotFound)
return
}
time.Sleep(120 * time.Millisecond)
atomic.AddInt32(&inflight, -1)
_, _ = w.Write([]byte(testLLMBody))
}))
defer srv.Close()
e := NewEngine(srv.URL, "test-model", "test-key")
e.lastDistill = time.Now() // 同上:本轮由测试显式驱动
const total = 9
for i := 0; i < total; i++ {
e.Enqueue(DistillInput{EpisodeID: "ep_serial_test", Content: "用户今天在鹤壁买了咖啡并读了一本关于马哲经济学的书,下午讨论了剪辑选型。"})
e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k")
if got := e.currentConcurrency(); got != 1 {
t.Fatalf("判据④未满足:/props 探测失败应保守取 1,实得 %d", got)
}
// 手动叠加触发 5 次:只有第一轮应真正执行,其余被 flushing 守卫挡掉
done := make(chan struct{})
go func() {
for i := 0; i < 5; i++ {
e.flush()
time.Sleep(10 * time.Millisecond)
}
// ⑤ 峰值并发真的被有效并发限住(用 2 槽,跑一轮 flush,峰值应 ≤2 且 ≥1)
func TestFlushPeakRespectsSlotMatchedConcurrency(t *testing.T) {
var peak, inflight int32
// 单独起桩(不复用 fakePropsServer),inflight 计数器用于判断「所有 LLM 调用已结束」
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/props":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"total_slots":2}`))
default:
// 记录峰值并发
n := atomic.AddInt32(&inflight, 1)
for {
p := atomic.LoadInt32(&peak)
if n <= p || atomic.CompareAndSwapInt32(&peak, p, n) {
break
}
}
time.Sleep(120 * time.Millisecond)
atomic.AddInt32(&inflight, -1)
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(testLLMBody))
}
close(done)
}()
<-done
// 等本轮真正结束(队列排空 + 计数落地)——守卫只挡重叠触发,**不应丢条目**
}))
defer srv.Close()
e := NewEngine(srv.URL+"/v1/chat/completions", "m", "k")
e.lastDistill = time.Now() // 抑制自动 flush,本轮显式驱动
const n = 6
for i := 0; i < n; i++ {
e.Enqueue(DistillInput{EpisodeID: "ep_ac2", Content: "用户今天在鹤壁买了咖啡并读了关于马哲经济学的书,下午讨论了剪辑软件选型。"})
}
e.flush() // 内部会 RefreshConcurrency(探测到 total_slots=2 → 并发=2)
// 等「所有 LLM 调用真正结束」(inflight 归 0)+ 计数落地,再读峰值。
// flush() 本身已在 wg.Wait() 后返回,但保险起见再确认 inflight=0。
deadline := time.Now().Add(5 * time.Second)
var used int
for time.Now().Before(deadline) {
e.mu.Lock()
used = e.dailyUsed
e.mu.Unlock()
if used == total && e.QueueLen() == 0 {
if atomic.LoadInt32(&inflight) == 0 && e.QueueLen() == 0 {
break
}
time.Sleep(20 * time.Millisecond)
time.Sleep(10 * time.Millisecond)
}
if used != total {
t.Fatalf("守卫丢条目:处理 %d/%d", used, total)
gotPeak := atomic.LoadInt32(&peak)
// 有效并发=2,峰值应 ≤2(不超过槽数)——这是「不打爆槽位」的硬上界
if gotPeak > 2 {
t.Fatalf("判据⑤未满足:峰值并发 %d 超出 2 槽匹配值(打爆了槽位)", gotPeak)
}
if p := atomic.LoadInt32(&peak); p > int32(distillConcurrency) {
t.Fatalf("重叠触发导致并发叠加:峰值=%d > %d", p, distillConcurrency)
// 峰值 ≥2(2 槽并发真的生效,不是串行)
if gotPeak < 2 {
t.Fatalf("判据⑤未满足:峰值并发 %d < 2(2 槽并发未生效)", gotPeak)
}
t.Logf("✅ 重叠触发 5 次:峰值并发 %d(上限 %d)、无丢条目(%d/%d)",
atomic.LoadInt32(&peak), distillConcurrency, used, total)
t.Logf("✅ AC2⑤: 2 槽匹配并发,峰值=%d(≤2 且 ≥2,不打爆槽位)", gotPeak)
}