rotate: add an optional random rotation feature in INT8/INT4 quantization method (#483)

Co-authored-by: rayx <rui.xing@alibaba-inc.com>
Co-authored-by: Jalin Wang <wangjianning.wjn@alibaba-inc.com>
Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com>
This commit is contained in:
Zhuanglin Zheng
2026-06-26 19:16:19 +08:00
committed by GitHub
co-authored by rayx Jalin Wang Copilot Autofix powered by AI
parent cdef8686c7
commit c54f5e16e9
54 changed files with 3605 additions and 179 deletions
@@ -1209,6 +1209,100 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID2) {
EXPECT_GT(topk1Recall, 0.80f);
}
// Test Flat + INT8 quantization + rotation end-to-end
TEST_F(FlatStreamerTest, TestInt8WithRotate) {
constexpr size_t kTestDim = 128;
constexpr size_t kCnt = 2000U;
constexpr size_t kTopk = 10;
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_NE(nullptr, streamer);
Params params;
IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, kTestDim);
index_meta_raw.set_metric("SquaredEuclidean", 0, Params());
// Create INT8 converter with rotation enabled
Params converter_params;
converter_params.set("integer_streaming.converter.enable_rotate", true);
auto converter = IndexFactory::CreateConverter("Int8StreamingConverter");
ASSERT_NE(nullptr, converter);
ASSERT_EQ(0, converter->init(index_meta_raw, converter_params));
IndexMeta index_meta = converter->meta();
auto reformer = IndexFactory::CreateReformer(index_meta.reformer_name());
ASSERT_NE(nullptr, reformer);
ASSERT_EQ(0, reformer->init(index_meta.reformer_params()));
Params stg_params;
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "TestInt8WithRotate.index", true));
ASSERT_EQ(0, streamer->init(index_meta, params));
ASSERT_EQ(0, streamer->open(storage));
// Add 2000 vectors
auto ctx = streamer->create_context();
ASSERT_TRUE(!!ctx);
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kTestDim);
std::mt19937 gen(42);
std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
for (size_t i = 0; i < kCnt; i++) {
NumericalVector<float> vec(kTestDim);
for (size_t j = 0; j < kTestDim; ++j) vec[j] = dist(gen);
std::string new_vec;
IndexQueryMeta new_meta;
ASSERT_EQ(0, reformer->convert(vec.data(), qmeta, &new_vec, &new_meta));
ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx));
}
streamer->flush(0UL);
streamer.reset();
storage.reset();
// Reopen: reformer should auto-detect rotator from storage
auto storage2 = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage2);
ASSERT_EQ(0, storage2->init(stg_params));
ASSERT_EQ(0, storage2->open(dir_ + "TestInt8WithRotate.index", false));
auto streamer2 = IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_NE(nullptr, streamer2);
ASSERT_EQ(0, streamer2->init(index_meta, params));
ASSERT_EQ(0, streamer2->open(storage2));
auto reformer2 = IndexFactory::CreateReformer(index_meta.reformer_name());
ASSERT_NE(nullptr, reformer2);
ASSERT_EQ(0, reformer2->init(index_meta.reformer_params()));
ASSERT_EQ(0, reformer2->load(storage2));
// Search: verify results are non-empty
auto knnCtx = streamer2->create_context();
knnCtx->set_topk(kTopk);
auto linearCtx = streamer2->create_context();
linearCtx->set_topk(kTopk);
NumericalVector<float> query(kTestDim);
for (size_t j = 0; j < kTestDim; ++j) query[j] = dist(gen);
std::string new_query;
IndexQueryMeta new_qmeta;
ASSERT_EQ(0,
reformer2->transform(query.data(), qmeta, &new_query, &new_qmeta));
ASSERT_EQ(0, streamer2->search_impl(new_query.data(), new_qmeta, knnCtx));
ASSERT_EQ(0,
streamer2->search_bf_impl(new_query.data(), new_qmeta, linearCtx));
EXPECT_EQ(kTopk, knnCtx->result().size());
EXPECT_EQ(kTopk, linearCtx->result().size());
}
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif