diff --git a/go/internal/consolidate/client.go b/go/internal/consolidate/client.go index 5e9f0d0..db0c277 100644 --- a/go/internal/consolidate/client.go +++ b/go/internal/consolidate/client.go @@ -74,15 +74,10 @@ func Run(dataDir, sqlitePath, mode string) (*Result, error) { LLMEndpoint: os.Getenv("LLM_ENDPOINT"), LLMModel: os.Getenv("LLM_MODEL"), LLMApiKey: os.Getenv("LLM_API_KEY"), - LLMBudget: 20, - // eps: 1024-dim BGE-M3 单位向量 - // euclidean² = 2(1-cosine), cosine = 1 - euclidean²/2 - // eps=0.5 → cosine>0.875 (very strict, high similarity) - // eps=1.0 → cosine>0.5 (moderate, semantic related) - // eps=1.5 → cosine>0 (trivial, almost all cluster → 1 group) - // 从 1 cluster 调整到有意义的语义聚类,从 0.5 开始试 - Epsilon: 0.1, - MinPoints: 3, + Epsilon: 1.0, + // 距离分布: p50=1.029, p95=1.135, max=1.184 + // eps=0.1 → 1435 noise (太严), eps=1.0 → 合理语义聚类, eps=1.5 → 1个巨大cluster + MinPoints: 3, ModelDir: os.Getenv("BGE_MODEL_DIR"), }) } diff --git a/rust/src/lancedb_ops.rs b/rust/src/lancedb_ops.rs index f27f5c3..3ee4e12 100644 --- a/rust/src/lancedb_ops.rs +++ b/rust/src/lancedb_ops.rs @@ -106,7 +106,7 @@ impl LanceDBOps { namespace: col_str(&batch, i, "namespace"), content: col_str(&batch, i, "content"), category: col_str(&batch, i, "category"), - vector: vec![], + vector: col_vector(&batch, i, "vector"), tier: col_str(&batch, i, "tier"), importance: col_f64(&batch, i, "importance"), quality_score: col_f64(&batch, i, "quality_score"), @@ -323,7 +323,7 @@ impl LanceDBOps { namespace: col_str(&batch, i, "namespace"), content: col_str(&batch, i, "content"), category: col_str(&batch, i, "category"), - vector: vec![], + vector: col_vector(&batch, i, "vector"), tier: col_str(&batch, i, "tier"), importance: col_f64(&batch, i, "importance"), quality_score: col_f64(&batch, i, "quality_score"), @@ -404,3 +404,27 @@ fn col_bool(b: &RecordBatch, r: usize, c: &str) -> bool { .map(|a| a.value(r)) .unwrap_or(false) } + +/// 读取 vector 列 (兼容 FixedSizeListArray 和 Float32Array) +fn col_vector(b: &RecordBatch, r: usize, c: &str) -> Vec { + let col = match b.column_by_name(c) { + Some(col) => col, + None => return vec![0.0_f32; 1024], + }; + // 优先尝试 FixedSizeListArray (当前使用的格式) + if let Some(list_arr) = col.as_any().downcast_ref::() { + let item_arr = list_arr.value(r); + if let Some(float_arr) = item_arr.as_any().downcast_ref::() { + return float_arr.values().to_vec(); + } + } + // 回退: Float32Array (展平格式) + if let Some(float_arr) = col.as_any().downcast_ref::() { + let stride = 1024; + let start = r * stride; + if start + stride <= float_arr.len() { + return float_arr.values()[start..start + stride].to_vec(); + } + } + vec![0.0_f32; 1024] +}