diff --git a/src/core/algorithm/hnsw_rabitq/hnsw_rabitq_streamer.cc b/src/core/algorithm/hnsw_rabitq/hnsw_rabitq_streamer.cc index 2ea2f6a..dea90c3 100644 --- a/src/core/algorithm/hnsw_rabitq/hnsw_rabitq_streamer.cc +++ b/src/core/algorithm/hnsw_rabitq/hnsw_rabitq_streamer.cc @@ -318,6 +318,16 @@ int HnswRabitqStreamer::open(IndexStorage::Pointer stg) { if (ret != 0) { return ret; } + + // Verify ex_bits consistency to avoid quantized data layout mismatch + if (reformer_->ex_bits() != entity_.ex_bits()) { + LOG_ERROR( + "ex_bits mismatch between reformer(%zu) and entity(%zu). " + "Reformer and entity must use the same total_bits configuration", + reformer_->ex_bits(), (size_t)entity_.ex_bits()); + return IndexError_Mismatch; + } + IndexMeta index_meta; ret = entity_.get_index_meta(&index_meta); if (ret == IndexError_NoExist) { diff --git a/src/core/algorithm/hnsw_rabitq/rabitq_reformer.cc b/src/core/algorithm/hnsw_rabitq/rabitq_reformer.cc index 977db8b..d8ffa95 100644 --- a/src/core/algorithm/hnsw_rabitq/rabitq_reformer.cc +++ b/src/core/algorithm/hnsw_rabitq/rabitq_reformer.cc @@ -85,6 +85,10 @@ size_t RabitqReformer::num_clusters() const { return impl_->num_clusters; } +size_t RabitqReformer::ex_bits() const { + return impl_->ex_bits; +} + RabitqMetricType RabitqReformer::rabitq_metric_type() const { return Impl::from_rabitq(impl_->metric_type); } diff --git a/src/core/algorithm/hnsw_rabitq/rabitq_reformer.h b/src/core/algorithm/hnsw_rabitq/rabitq_reformer.h index 3977a61..72bdc53 100644 --- a/src/core/algorithm/hnsw_rabitq/rabitq_reformer.h +++ b/src/core/algorithm/hnsw_rabitq/rabitq_reformer.h @@ -61,6 +61,7 @@ class RabitqReformer : public IndexReformer { HnswRabitqQueryEntity *entity) const; size_t num_clusters() const; + size_t ex_bits() const; RabitqMetricType rabitq_metric_type() const; private: diff --git a/tests/core/algorithm/hnsw_rabitq/hnsw_rabitq_streamer_test.cc b/tests/core/algorithm/hnsw_rabitq/hnsw_rabitq_streamer_test.cc index 231dd7b..1e58276 100644 --- a/tests/core/algorithm/hnsw_rabitq/hnsw_rabitq_streamer_test.cc +++ b/tests/core/algorithm/hnsw_rabitq/hnsw_rabitq_streamer_test.cc @@ -614,5 +614,47 @@ TEST_F(HnswRabitqStreamerTest, TestDimensions) { } } +TEST_F(HnswRabitqStreamerTest, TestExBitsMismatch) { + auto holder = + make_shared>(dim); + size_t doc_cnt = 500UL; + for (size_t i = 0; i < doc_cnt; i++) { + NumericalVector vec(dim); + for (size_t j = 0; j < dim; ++j) { + vec[j] = static_cast(i * dim + j) / 1000.0f; + } + ASSERT_TRUE(holder->emplace(i, vec)); + } + + // Train converter with default total_bits (7, ex_bits=6) + RabitqConverter converter; + converter.init(*index_meta_ptr_, ailego::Params()); + ASSERT_EQ(converter.train(holder), 0); + std::shared_ptr index_reformer; + ASSERT_EQ(converter.to_reformer(&index_reformer), 0); + auto reformer = std::dynamic_pointer_cast(index_reformer); + + // Create streamer with total_bits=2 (ex_bits=1), mismatching the reformer + IndexStreamer::Pointer streamer = + std::make_shared(holder, reformer); + + ailego::Params params; + params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U); + params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U); + params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U); + params.set("proxima.hnsw_rabitq.general.dimension", dim); + params.set(PARAM_RABITQ_TOTAL_BITS, 2U); + ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); + + auto storage = IndexFactory::CreateStorage("MMapFileStorage"); + ASSERT_NE(nullptr, storage); + ailego::Params stg_params; + ASSERT_EQ(0, storage->init(stg_params)); + ASSERT_EQ(0, storage->open(dir_ + "/TestExBitsMismatch", true)); + + // open() should detect ex_bits mismatch and return IndexError_Mismatch + ASSERT_EQ(IndexError_Mismatch, streamer->open(storage)); +} + } // namespace core } // namespace zvec diff --git a/tests/core/interface/index_interface_test.cc b/tests/core/interface/index_interface_test.cc index 8d3adbd..44d9061 100644 --- a/tests/core/interface/index_interface_test.cc +++ b/tests/core/interface/index_interface_test.cc @@ -1697,6 +1697,16 @@ TEST(IndexInterface, HNSWRabitqGeneral) { .build()); // HNSWRabitq with custom total_bits + // Reformer must be re-created with matching total_bits to keep ex_bits + // consistent between reformer and entity. + RabitqConverter converter2; + Params converter2_params; + converter2_params.set(PARAM_RABITQ_TOTAL_BITS, 2u); + converter2.init(*index_meta_ptr_, converter2_params); + ASSERT_EQ(converter2.train(holder), 0); + std::shared_ptr index_reformer2; + ASSERT_EQ(converter2.to_reformer(&index_reformer2), 0); + func(HNSWRabitqIndexParamBuilder() .WithMetricType(MetricType::kL2sq) .WithDataType(DataType::DT_FP32) @@ -1705,7 +1715,7 @@ TEST(IndexInterface, HNSWRabitqGeneral) { .WithEFConstruction(100) .WithTotalBits(2) .WithProvider(holder) - .WithReformer(index_reformer) + .WithReformer(index_reformer2) .Build(), HNSWRabitqQueryParamBuilder() .with_topk(10) @@ -1729,53 +1739,55 @@ TEST(IndexInterface, ContiguousMemoryEndToEnd) { // build_then_search builds an index from scratch (with use_contiguous_memory // possibly enabled), closes it, then reopens with the same params and runs a // search for each inserted vector, asserting top-1 is itself. - auto build_then_search = [&](const BaseIndexParam::Pointer ¶m, - const BaseIndexQueryParam::Pointer &query_param) { - zvec::test_util::RemoveTestFiles(index_name); + auto build_then_search = + [&](const BaseIndexParam::Pointer ¶m, + const BaseIndexQueryParam::Pointer &query_param) { + zvec::test_util::RemoveTestFiles(index_name); - // Phase 1: build & persist. - { - auto index = IndexFactory::CreateAndInitIndex(*param); - ASSERT_NE(nullptr, index); - ASSERT_EQ(0, index->Open(index_name, - {StorageOptions::StorageType::kMMAP, true})); + // Phase 1: build & persist. + { + auto index = IndexFactory::CreateAndInitIndex(*param); + ASSERT_NE(nullptr, index); + ASSERT_EQ(0, index->Open(index_name, + {StorageOptions::StorageType::kMMAP, true})); - std::vector vec(kDimension); - for (uint32_t i = 0; i < kNumDocs; ++i) { - for (uint32_t d = 0; d < kDimension; ++d) { - vec[d] = static_cast(i); + std::vector vec(kDimension); + for (uint32_t i = 0; i < kNumDocs; ++i) { + for (uint32_t d = 0; d < kDimension; ++d) { + vec[d] = static_cast(i); + } + VectorData data{DenseVector{vec.data()}}; + ASSERT_EQ(0, index->Add(data, i)); + } + ASSERT_EQ(0, index->Train()); + ASSERT_EQ(0, index->Close()); } - VectorData data{DenseVector{vec.data()}}; - ASSERT_EQ(0, index->Add(data, i)); - } - ASSERT_EQ(0, index->Train()); - ASSERT_EQ(0, index->Close()); - } - // Phase 2: reopen with same params (contiguous memory takes effect here) - // and search. - { - auto index = IndexFactory::CreateAndInitIndex(*param); - ASSERT_NE(nullptr, index); - ASSERT_EQ(0, index->Open(index_name, - {StorageOptions::StorageType::kMMAP, false})); + // Phase 2: reopen with same params (contiguous memory takes effect + // here) and search. + { + auto index = IndexFactory::CreateAndInitIndex(*param); + ASSERT_NE(nullptr, index); + ASSERT_EQ(0, + index->Open(index_name, + {StorageOptions::StorageType::kMMAP, false})); - std::vector q(kDimension); - for (uint32_t i = 0; i < kNumDocs; i += 50) { - for (uint32_t d = 0; d < kDimension; ++d) { - q[d] = static_cast(i); + std::vector q(kDimension); + for (uint32_t i = 0; i < kNumDocs; i += 50) { + for (uint32_t d = 0; d < kDimension; ++d) { + q[d] = static_cast(i); + } + VectorData query{DenseVector{q.data()}}; + SearchResult result; + ASSERT_EQ(0, index->Search(query, query_param, &result)); + ASSERT_GT(result.doc_list_.size(), 0UL); + ASSERT_EQ(i, result.doc_list_[0].key()); + } + ASSERT_EQ(0, index->Close()); } - VectorData query{DenseVector{q.data()}}; - SearchResult result; - ASSERT_EQ(0, index->Search(query, query_param, &result)); - ASSERT_GT(result.doc_list_.size(), 0UL); - ASSERT_EQ(i, result.doc_list_[0].key()); - } - ASSERT_EQ(0, index->Close()); - } - zvec::test_util::RemoveTestFiles(index_name); - }; + zvec::test_util::RemoveTestFiles(index_name); + }; // HNSW + use_contiguous_memory=true build_then_search(HNSWIndexParamBuilder()