From d29bffca3972b86fe9309695ffc40856d9bc9156 Mon Sep 17 00:00:00 2001
From: Zihao Wang <47910959+JoeJRW@users.noreply.github.com>
Date: Thu, 16 Apr 2026 10:07:14 +0800
Subject: [PATCH] fix(quantizer): use rounded int8 values for SQ8 metadata to
 fix recall drop (#329)

fixes #328

Problem:
SQ8 metadata (squared_sum, sum) was computed from pre-rounded float
values, causing mismatch with actual stored int8 values. On asymmetric
datasets (e.g. OpenAI 1536D where |x_min| >> x_max), this leads to
severe recall drop.

Solution:
Move std::round before accumulating squared_sum and sum.

Co-authored-by: rayx <rui.xing@alibaba-inc.com>
---
 src/core/quantizer/record_quantizer.h           | 5 ++---
 tests/core/algorithm/hnsw/hnsw_streamer_test.cc | 4 ++--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/core/quantizer/record_quantizer.h b/src/core/quantizer/record_quantizer.h
index b1095a2..e107eeb 100644
--- a/src/core/quantizer/record_quantizer.h
+++ b/src/core/quantizer/record_quantizer.h
@@ -44,11 +44,10 @@ class RecordQuantizer {
         scale = 254 / std::max(max - min, epsilon);
         bias = -min * scale - 127;
         for (size_t i = 0; i < dim; ++i) {
-          float v = vec[i] * scale + bias;
+          float v = std::round(vec[i] * scale + bias);
           squared_sum += v * v;
           sum += v;
-          (reinterpret_cast<int8_t *>(out))[i] =
-              static_cast<int8_t>(std::round(v));
+          (reinterpret_cast<int8_t *>(out))[i] = static_cast<int8_t>(v);
           int8_sum += (reinterpret_cast<int8_t *>(out))[i];
         }
         extras = reinterpret_cast<float *>(static_cast<int8_t *>(out) + dim);
diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc
index 3f27f52..47560fd 100644
--- a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc
+++ b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc
@@ -2772,7 +2772,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt8Converter) {
   for (size_t i = 0; i < cnt; i++) {
     float add_on = i * 10;
     for (size_t j = 0; j < dim; ++j) {
-      if (j < dim / 4)
+      if (j < 3 * dim / 4)
         vec[j] = fixed_value;
       else
         vec[j] = fixed_value + add_on;
@@ -2812,7 +2812,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt8Converter) {
   for (size_t i = 0; i < query_cnt; i++) {
     float add_on = i * 10;
     for (size_t j = 0; j < dim; ++j) {
-      if (j < dim / 4)
+      if (j < 3 * dim / 4)
         vec[j] = fixed_value;
       else
         vec[j] = fixed_value + add_on;