fix: use per-block filter instead of per-segment filter during query (#223)

* fix: id mismatch

* fix: remove debug string

* fix: use protected

* chore(comment): add comments

* feat: more ut

* Update vector_recall_test.cc

* fix: resolve comments

src/db/index/column/vector_column/combined_vector_column_indexer.cc

@@ -104,11 +104,22 @@ Result<IndexResults::Ptr> CombinedVectorColumnIndexer::Search(
       need_refine = true;
     }
 
+    const IndexFilter *filter{nullptr};
+    auto per_block_filter =
+        BlockOffsetFilter{query_params.filter, block_offsets_[i]};
+    if (query_params.filter) {
+      if (block_offsets_[i] > 0) {
+        filter = &per_block_filter;
+      } else {
+        filter = query_params.filter;
+      }
+    }
+
     vector_column_params::QueryParams modified_query_params{
         query_params.data_type,
         query_params.dimension,
         query_params.topk,
-        query_params.filter,
+        filter,
         query_params.fetch_vector,
         query_params.query_params,
         query_params.group_by

src/db/index/column/vector_column/combined_vector_column_indexer.h

@@ -15,9 +15,9 @@
 
 #include <memory>
 #include <vector>
+#include "db/index/common/index_filter.h"
 #include "vector_column_indexer.h"
 #include "vector_column_params.h"
-#include "vector_index_results.h"
 
 namespace zvec {
 
@@ -42,8 +42,31 @@ class CombinedVectorColumnIndexer {
   virtual Result<vector_column_params::VectorDataBuffer> Fetch(
       uint32_t segment_doc_id) const;
 
-  // for ut
+
  protected:
+  /**
+   * A filter wrapper that applies an offset to document IDs before
+   * delegating to an inner filter.
+   *
+   * This is used when multiple blocks with different ID offsets are stored.
+   * Each block has its own local ID space, and this filter translates
+   * block-level IDs to segment-level IDs before checking the inner filter.
+   */
+  class BlockOffsetFilter : public IndexFilter {
+   public:
+    BlockOffsetFilter(const IndexFilter *inner_filter, uint64_t offset)
+        : inner_filter_(inner_filter), offset_(offset) {}
+
+    bool is_filtered(uint64_t id) const override {
+      return inner_filter_->is_filtered(id + offset_);
+    }
+
+   private:
+    const IndexFilter *inner_filter_;
+    uint64_t offset_;
+  };
+
+  // for ut
   CombinedVectorColumnIndexer() = default;
 
 

src/db/index/segment/segment.cc

@@ -302,14 +302,14 @@ class SegmentImpl : public Segment,
   void fresh_persist_chunked_array();
 
  private:
-  // scalar forward
+  // scalar forward (uses segment-local doc ID)
   MemForwardStore::Ptr memory_store_;
   std::vector<BaseForwardStore::Ptr> persist_stores_;
 
-  // scalar index
+  // scalar index (uses segment-local doc ID)
   InvertedIndexer::Ptr invert_indexers_;
 
-  // vector index
+  // vector index (uses block-local doc ID, each indexer starts from 0)
   std::unordered_map<std::string, VectorColumnIndexer::Ptr>
       memory_vector_indexers_;
 
@@ -339,7 +339,7 @@ class SegmentImpl : public Segment,
   IDMap::Ptr id_map_;
   DeleteStore::Ptr delete_store_;
 
-  // local_id(index) -> global_doc_id(value)
+  // Maps segment-local doc ID (array index) to global doc ID (stored value)
   std::vector<uint64_t> doc_ids_;
 
   std::array<std::variant<std::vector<int>,

tests/db/sqlengine/recall_base.h

@@ -253,6 +253,7 @@ inline Segment::Ptr RecallTest::create_segment() {
   SegmentOptions options;
   options.read_only_ = false;
   options.enable_mmap_ = true;
+  options.max_buffer_size_ = 256 * 1024;
 
   auto result =
       Segment::CreateAndOpen(GetPath(), *collection_schema_, 0, 0, id_map,

tests/db/sqlengine/vector_recall_test.cc

@@ -212,4 +212,71 @@ TEST_F(VectorRecallTest, Sparse) {
   }
 }
 
-}  // namespace zvec::sqlengine
+TEST_F(VectorRecallTest, DeleteFilter) {
+  // This test uses only one segment and thus we only operate on the first one
+  for (int i = 0; i < 4000; i++) {
+    segments_[0]->Delete("pk_" + std::to_string(i));
+  }
+
+  VectorQuery query;
+  query.output_fields_ = {"name", "age"};
+  query.topk_ = 100;
+  std::vector<float> feature(4, 0.0);
+  query.query_vector_.assign((const char *)feature.data(),
+                             feature.size() * sizeof(float));
+  query.field_name_ = "dense";
+
+  auto engine = SQLEngine::create(std::make_shared<Profiler>());
+  auto ret = engine->execute(collection_schema_, query, segments_);
+  if (!ret) {
+    LOG_ERROR("execute failed: [%s]", ret.error().c_str());
+  }
+  ASSERT_TRUE(ret.has_value());
+  auto docs = ret.value();
+  EXPECT_EQ(docs.size(), 100);
+  for (size_t j = 0; j < docs.size(); j++) {
+    auto &doc = docs[j];
+    int doc_id = j + 4000;
+    EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
+    auto age = doc->get<int32_t>("age");
+    EXPECT_EQ(age.value(), doc_id % 100);
+    auto name = doc->get<std::string>("name");
+    ASSERT_TRUE(name);
+    EXPECT_EQ(name.value(), "user_" + std::to_string(doc_id % 100));
+    EXPECT_FLOAT_EQ(doc->score(), (float)doc_id * doc_id * 4);
+  }
+}
+
+TEST_F(VectorRecallTest, HybridInvertForwardDeleteFilter) {
+  // In previous test, docs[0-4000) has been deleted
+  VectorQuery query;
+  query.output_fields_ = {"name", "age"};
+  query.filter_ = "invert_id >= 6000 and id < 6080";
+  query.topk_ = 100;
+  std::vector<float> feature(4, 0.0);
+  query.query_vector_.assign((const char *)feature.data(),
+                             feature.size() * sizeof(float));
+  query.field_name_ = "dense";
+
+  auto engine = SQLEngine::create(std::make_shared<Profiler>());
+  auto ret = engine->execute(collection_schema_, query, segments_);
+  if (!ret) {
+    LOG_ERROR("execute failed: [%s]", ret.error().c_str());
+  }
+  ASSERT_TRUE(ret.has_value());
+  auto docs = ret.value();
+  EXPECT_EQ(docs.size(), 80);
+  for (size_t j = 0; j < docs.size(); j++) {
+    auto &doc = docs[j];
+    int doc_id = j + 6000;
+    EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
+    auto age = doc->get<int32_t>("age");
+    EXPECT_EQ(age.value(), doc_id % 100);
+    auto name = doc->get<std::string>("name");
+    ASSERT_TRUE(name);
+    EXPECT_EQ(name.value(), "user_" + std::to_string(doc_id % 100));
+    EXPECT_FLOAT_EQ(doc->score(), (float)doc_id * doc_id * 4);
+  }
+}
+
+}  // namespace zvec::sqlengine