diff --git a/src/core/quantizer/record_quantizer.h b/src/core/quantizer/record_quantizer.h index b1095a2..e107eeb 100644 --- a/src/core/quantizer/record_quantizer.h +++ b/src/core/quantizer/record_quantizer.h @@ -44,11 +44,10 @@ class RecordQuantizer { scale = 254 / std::max(max - min, epsilon); bias = -min * scale - 127; for (size_t i = 0; i < dim; ++i) { - float v = vec[i] * scale + bias; + float v = std::round(vec[i] * scale + bias); squared_sum += v * v; sum += v; - (reinterpret_cast(out))[i] = - static_cast(std::round(v)); + (reinterpret_cast(out))[i] = static_cast(v); int8_sum += (reinterpret_cast(out))[i]; } extras = reinterpret_cast(static_cast(out) + dim); diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc index 3f27f52..47560fd 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc @@ -2772,7 +2772,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt8Converter) { for (size_t i = 0; i < cnt; i++) { float add_on = i * 10; for (size_t j = 0; j < dim; ++j) { - if (j < dim / 4) + if (j < 3 * dim / 4) vec[j] = fixed_value; else vec[j] = fixed_value + add_on; @@ -2812,7 +2812,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt8Converter) { for (size_t i = 0; i < query_cnt; i++) { float add_on = i * 10; for (size_t j = 0; j < dim; ++j) { - if (j < dim / 4) + if (j < 3 * dim / 4) vec[j] = fixed_value; else vec[j] = fixed_value + add_on;