fix: ivf provider sorted by local id (#422)

This commit is contained in:
ZeFeng Yin
2026-05-26 10:33:44 +08:00
committed by GitHub
parent bdf58fc2d2
commit d9b0920ac7
4 changed files with 88 additions and 8 deletions
+13
View File
@@ -171,6 +171,19 @@ class IVFEntity {
return *static_cast<const uint64_t *>(data);
}
//! Retrieve the key-order mapping (sorted rank -> local_id).
//! mapping[rank] is the local_id of the vector with the rank-th smallest
//! key. Returns nullptr if mapping segment is unavailable.
const uint32_t *get_key_order_mapping() const {
if (!mapping_) return nullptr;
const void *data = nullptr;
const size_t size = vector_count() * sizeof(uint32_t);
if (mapping_->read(0, &data, size) != size) {
return nullptr;
}
return static_cast<const uint32_t *>(data);
}
//! Retrieve vector by local id
const void *get_vector(size_t id) const;
+55 -3
View File
@@ -13,6 +13,9 @@
// limitations under the License.
#pragma once
#include <algorithm>
#include <numeric>
#include <vector>
#include <zvec/core/framework/index_searcher.h>
#include "ivf_entity.h"
@@ -33,7 +36,7 @@ class IVFIndexProvider : public IndexProvider {
public:
//! Create a new iterator
virtual Iterator::Pointer create_iterator(void) override {
return Iterator::Pointer(new (std::nothrow) Iterator(entity_));
return Iterator::Pointer(new (std::nothrow) SortedIterator(entity_));
}
//! Retrieve count of vectors
@@ -67,13 +70,62 @@ class IVFIndexProvider : public IndexProvider {
}
private:
class SortedIterator : public IndexProvider::Iterator {
public:
SortedIterator(const IVFEntity::Pointer &entity) : entity_(entity) {
count_ = entity_->vector_count();
mapping_ = entity_->get_key_order_mapping();
if (!mapping_) {
// Fallback: compute sorting if mapping segment is unavailable
fallback_.resize(count_);
std::iota(fallback_.begin(), fallback_.end(), size_t(0));
std::sort(fallback_.begin(), fallback_.end(), [&](size_t a, size_t b) {
return entity_->get_key(a) < entity_->get_key(b);
});
}
}
//! Retrieve pointer of data
//! NOTICE: the vec feature will be changed after iterating to next, so
//! the caller need to keep a copy of it before iterator to next vector
virtual const void *data(void) const override {
return entity_->get_vector(current_local_id());
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
return pos_ < count_;
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
return entity_->get_key(current_local_id());
}
//! Next iterator
virtual void next(void) override {
++pos_;
}
private:
size_t current_local_id() const {
return mapping_ ? static_cast<size_t>(mapping_[pos_]) : fallback_[pos_];
}
//! Members
IVFEntity::Pointer entity_;
const uint32_t *mapping_{nullptr}; // points into mapping_ segment data
std::vector<size_t> fallback_; // used only if mapping_ unavailable
size_t count_{0};
size_t pos_{0};
};
//! Original sequential iterator (kept for potential internal use)
class Iterator : public IndexProvider::Iterator {
public:
Iterator(const IVFEntity::Pointer &entity) : entity_(entity) {}
//! Retrieve pointer of data
//! NOTICE: the vec feature will be changed after iterating to next, so
//! the caller need to keep a copy of it before iterator to next vector
virtual const void *data(void) const override {
return entity_->get_vector(index_);
}