xiaoxue_admin
/
zvec
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(quantizer): use rounded int8 values for SQ8 metadata to fix recall drop (#329) 

fixes #328

Problem:
SQ8 metadata (squared_sum, sum) was computed from pre-rounded float
values, causing mismatch with actual stored int8 values. On asymmetric
datasets (e.g. OpenAI 1536D where |x_min| >> x_max), this leads to
severe recall drop.

Solution:
Move std::round before accumulating squared_sum and sum.

Co-authored-by: rayx <rui.xing@alibaba-inc.com>
This commit is contained in:
Zihao Wang 2026-04-16 10:07:14 +08:00 committed by GitHub
parent 6737810190
commit d29bffca39
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/core/quantizer/record_quantizer.h
View File

@ -44,11 +44,10 @@ class RecordQuantizer {
scale = 254 / std::max(max - min, epsilon);
bias = -min * scale - 127;
for (size_t i = 0; i < dim; ++i) {
float v = vec[i] * scale + bias;
float v = std::round(vec[i] * scale + bias);
squared_sum += v * v;
sum += v;
(reinterpret_cast<int8_t *>(out))[i] =
static_cast<int8_t>(std::round(v));
(reinterpret_cast<int8_t *>(out))[i] = static_cast<int8_t>(v);
int8_sum += (reinterpret_cast<int8_t *>(out))[i];
}
extras = reinterpret_cast<float *>(static_cast<int8_t *>(out) + dim);

4
tests/core/algorithm/hnsw/hnsw_streamer_test.cc
View File

@ -2772,7 +2772,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt8Converter) {
for (size_t i = 0; i < cnt; i++) {
float add_on = i * 10;
for (size_t j = 0; j < dim; ++j) {
if (j < dim / 4)
if (j < 3 * dim / 4)
vec[j] = fixed_value;
else
vec[j] = fixed_value + add_on;
@ -2812,7 +2812,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt8Converter) {
for (size_t i = 0; i < query_cnt; i++) {
float add_on = i * 10;
for (size_t j = 0; j < dim; ++j) {
if (j < dim / 4)
if (j < 3 * dim / 4)
vec[j] = fixed_value;
else
vec[j] = fixed_value + add_on;