From 9314cb14a636d6fb5d46ff4311851be4f5d32072 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Mon, 16 Mar 2026 10:55:51 +0800 Subject: [PATCH] fix: use per-block filter instead of per-segment filter during query (#223) * fix: id mismatch * fix: remove debug string * fix: use protected * chore(comment): add comments * feat: more ut * Update vector_recall_test.cc * fix: resolve comments --- .../combined_vector_column_indexer.cc | 13 +++- .../combined_vector_column_indexer.h | 27 +++++++- src/db/index/segment/segment.cc | 8 +-- tests/db/sqlengine/recall_base.h | 1 + tests/db/sqlengine/vector_recall_test.cc | 69 ++++++++++++++++++- 5 files changed, 110 insertions(+), 8 deletions(-) diff --git a/src/db/index/column/vector_column/combined_vector_column_indexer.cc b/src/db/index/column/vector_column/combined_vector_column_indexer.cc index 70c71d0..14fd219 100644 --- a/src/db/index/column/vector_column/combined_vector_column_indexer.cc +++ b/src/db/index/column/vector_column/combined_vector_column_indexer.cc @@ -104,11 +104,22 @@ Result CombinedVectorColumnIndexer::Search( need_refine = true; } + const IndexFilter *filter{nullptr}; + auto per_block_filter = + BlockOffsetFilter{query_params.filter, block_offsets_[i]}; + if (query_params.filter) { + if (block_offsets_[i] > 0) { + filter = &per_block_filter; + } else { + filter = query_params.filter; + } + } + vector_column_params::QueryParams modified_query_params{ query_params.data_type, query_params.dimension, query_params.topk, - query_params.filter, + filter, query_params.fetch_vector, query_params.query_params, query_params.group_by diff --git a/src/db/index/column/vector_column/combined_vector_column_indexer.h b/src/db/index/column/vector_column/combined_vector_column_indexer.h index b0b0589..9235791 100644 --- a/src/db/index/column/vector_column/combined_vector_column_indexer.h +++ b/src/db/index/column/vector_column/combined_vector_column_indexer.h @@ -15,9 +15,9 @@ #include #include +#include "db/index/common/index_filter.h" #include "vector_column_indexer.h" #include "vector_column_params.h" -#include "vector_index_results.h" namespace zvec { @@ -42,8 +42,31 @@ class CombinedVectorColumnIndexer { virtual Result Fetch( uint32_t segment_doc_id) const; - // for ut + protected: + /** + * A filter wrapper that applies an offset to document IDs before + * delegating to an inner filter. + * + * This is used when multiple blocks with different ID offsets are stored. + * Each block has its own local ID space, and this filter translates + * block-level IDs to segment-level IDs before checking the inner filter. + */ + class BlockOffsetFilter : public IndexFilter { + public: + BlockOffsetFilter(const IndexFilter *inner_filter, uint64_t offset) + : inner_filter_(inner_filter), offset_(offset) {} + + bool is_filtered(uint64_t id) const override { + return inner_filter_->is_filtered(id + offset_); + } + + private: + const IndexFilter *inner_filter_; + uint64_t offset_; + }; + + // for ut CombinedVectorColumnIndexer() = default; diff --git a/src/db/index/segment/segment.cc b/src/db/index/segment/segment.cc index 2d03cd7..43928f3 100644 --- a/src/db/index/segment/segment.cc +++ b/src/db/index/segment/segment.cc @@ -302,14 +302,14 @@ class SegmentImpl : public Segment, void fresh_persist_chunked_array(); private: - // scalar forward + // scalar forward (uses segment-local doc ID) MemForwardStore::Ptr memory_store_; std::vector persist_stores_; - // scalar index + // scalar index (uses segment-local doc ID) InvertedIndexer::Ptr invert_indexers_; - // vector index + // vector index (uses block-local doc ID, each indexer starts from 0) std::unordered_map memory_vector_indexers_; @@ -339,7 +339,7 @@ class SegmentImpl : public Segment, IDMap::Ptr id_map_; DeleteStore::Ptr delete_store_; - // local_id(index) -> global_doc_id(value) + // Maps segment-local doc ID (array index) to global doc ID (stored value) std::vector doc_ids_; std::array, diff --git a/tests/db/sqlengine/recall_base.h b/tests/db/sqlengine/recall_base.h index 8c2a88a..3e45714 100644 --- a/tests/db/sqlengine/recall_base.h +++ b/tests/db/sqlengine/recall_base.h @@ -253,6 +253,7 @@ inline Segment::Ptr RecallTest::create_segment() { SegmentOptions options; options.read_only_ = false; options.enable_mmap_ = true; + options.max_buffer_size_ = 256 * 1024; auto result = Segment::CreateAndOpen(GetPath(), *collection_schema_, 0, 0, id_map, diff --git a/tests/db/sqlengine/vector_recall_test.cc b/tests/db/sqlengine/vector_recall_test.cc index 597b63a..d3dbccd 100644 --- a/tests/db/sqlengine/vector_recall_test.cc +++ b/tests/db/sqlengine/vector_recall_test.cc @@ -212,4 +212,71 @@ TEST_F(VectorRecallTest, Sparse) { } } -} // namespace zvec::sqlengine \ No newline at end of file +TEST_F(VectorRecallTest, DeleteFilter) { + // This test uses only one segment and thus we only operate on the first one + for (int i = 0; i < 4000; i++) { + segments_[0]->Delete("pk_" + std::to_string(i)); + } + + VectorQuery query; + query.output_fields_ = {"name", "age"}; + query.topk_ = 100; + std::vector feature(4, 0.0); + query.query_vector_.assign((const char *)feature.data(), + feature.size() * sizeof(float)); + query.field_name_ = "dense"; + + auto engine = SQLEngine::create(std::make_shared()); + auto ret = engine->execute(collection_schema_, query, segments_); + if (!ret) { + LOG_ERROR("execute failed: [%s]", ret.error().c_str()); + } + ASSERT_TRUE(ret.has_value()); + auto docs = ret.value(); + EXPECT_EQ(docs.size(), 100); + for (size_t j = 0; j < docs.size(); j++) { + auto &doc = docs[j]; + int doc_id = j + 4000; + EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id)); + auto age = doc->get("age"); + EXPECT_EQ(age.value(), doc_id % 100); + auto name = doc->get("name"); + ASSERT_TRUE(name); + EXPECT_EQ(name.value(), "user_" + std::to_string(doc_id % 100)); + EXPECT_FLOAT_EQ(doc->score(), (float)doc_id * doc_id * 4); + } +} + +TEST_F(VectorRecallTest, HybridInvertForwardDeleteFilter) { + // In previous test, docs[0-4000) has been deleted + VectorQuery query; + query.output_fields_ = {"name", "age"}; + query.filter_ = "invert_id >= 6000 and id < 6080"; + query.topk_ = 100; + std::vector feature(4, 0.0); + query.query_vector_.assign((const char *)feature.data(), + feature.size() * sizeof(float)); + query.field_name_ = "dense"; + + auto engine = SQLEngine::create(std::make_shared()); + auto ret = engine->execute(collection_schema_, query, segments_); + if (!ret) { + LOG_ERROR("execute failed: [%s]", ret.error().c_str()); + } + ASSERT_TRUE(ret.has_value()); + auto docs = ret.value(); + EXPECT_EQ(docs.size(), 80); + for (size_t j = 0; j < docs.size(); j++) { + auto &doc = docs[j]; + int doc_id = j + 6000; + EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id)); + auto age = doc->get("age"); + EXPECT_EQ(age.value(), doc_id % 100); + auto name = doc->get("name"); + ASSERT_TRUE(name); + EXPECT_EQ(name.value(), "user_" + std::to_string(doc_id % 100)); + EXPECT_FLOAT_EQ(doc->score(), (float)doc_id * doc_id * 4); + } +} + +} // namespace zvec::sqlengine