Initial commit

This commit is contained in:
sanyi
2025-12-30 11:02:17 +08:00
commit 6524fabd80
841 changed files with 293919 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
cc_library(
NAME core_knn_ivf STATIC SHARED STRICT ALWAYS_LINK
SRCS *.cc
LIBS zvec_ailego core_framework core_knn_cluster
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
VERSION "${PROXIMA_ZVEC_VERSION}"
)
+829
View File
@@ -0,0 +1,829 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ivf_builder.h"
#include <ailego/pattern/defer.h>
#include <ailego/utility/string_helper.h>
#include "cluster/cluster_params.h"
#include "ivf_dumper.h"
namespace zvec {
namespace core {
/*! IndexHolder support filtered by vector labels
*/
class LabelFilteredIndexHolder : public IndexHolder {
public:
/*! Index Holder Iterator
*/
class Iterator : public IndexHolder::Iterator {
public:
//! Index Holder Iterator Pointer
typedef std::unique_ptr<Iterator> Pointer;
//! Constructor
Iterator(const IVFBuilder::RandomAccessIndexHolder::Pointer &holder,
const std::vector<uint32_t> *elems)
: holder_(holder), elems_(elems) {}
//! Destructor
virtual ~Iterator(void) {}
//! Retrieve pointer of data
virtual const void *data(void) const override {
return holder_->element((*elems_)[index_]);
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
return index_ < elems_->size();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
return (*elems_)[index_];
}
//! Next iterator
virtual void next(void) override {
++index_;
}
private:
//! Members
const IVFBuilder::RandomAccessIndexHolder::Pointer holder_{nullptr};
const std::vector<uint32_t> *elems_{nullptr};
size_t index_{0};
};
//! Constructor
LabelFilteredIndexHolder(
const IVFBuilder::RandomAccessIndexHolder::Pointer &holder,
const std::vector<uint32_t> &items)
: holder_(holder), elems_(&items) {}
//! Retrieve count of elements in holder (-1 indicates unknown)
virtual size_t count(void) const override {
return elems_->size();
}
//! Retrieve dimension
virtual size_t dimension(void) const override {
return holder_->dimension();
}
//! Retrieve type information
virtual IndexMeta::DataType data_type(void) const override {
return holder_->data_type();
}
//! Retrieve element size in bytes
virtual size_t element_size(void) const override {
return holder_->element_size();
}
//! Retrieve if it can multi-pass
virtual bool multipass(void) const override {
return true;
}
//! Create a new iterator
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
return IndexHolder::Iterator::Pointer(
new LabelFilteredIndexHolder::Iterator(holder_, elems_));
}
private:
//! Members
const IVFBuilder::RandomAccessIndexHolder::Pointer holder_{};
const std::vector<uint32_t> *elems_{};
};
IVFBuilder::IVFBuilder() {}
IVFBuilder::~IVFBuilder() {
this->cleanup();
}
int IVFBuilder::init(const IndexMeta &meta, const ailego::Params &params) {
LOG_INFO("Begin IVFBuilder::init!");
if (state_ != INIT) {
LOG_ERROR("IVFBuilder state wrong. state=%d", state_);
return IndexError_Logic;
}
meta_ = meta;
converted_meta_ = meta;
quantized_meta_ = meta;
// Clear the converter/reformer params for external transforms
converted_meta_.set_reformer(std::string(), 0, ailego::Params());
converted_meta_.set_converter(std::string(), 0, ailego::Params());
quantized_meta_.set_reformer(std::string(), 0, ailego::Params());
quantized_meta_.set_converter(std::string(), 0, ailego::Params());
params_ = params;
if (!IndexFactory::HasMetric(meta_.metric_name())) {
LOG_ERROR("Metric %s not exist", meta_.metric_name().c_str());
return IndexError_NoExist;
}
int ret = parse_centroids_num(params);
ivf_check_with_msg(ret, "Failed to parse centroids, ret=%d", ret);
ret = parse_clustering_params(params);
ivf_check_with_msg(ret, "Failed to parse clustering params, ret=%d", ret);
ret = parse_general_params(params);
ivf_check_with_msg(ret, "Failed to parse general params, ret=%d", ret);
LOG_INFO("End IVFBuilder::init!");
LOG_DEBUG(
"Converter=%s Quantizer=%s Optimizer=%s "
"OptimizerQuantizer=%s QuantizeByCentroid=%u StoreFeatures=%u "
"ClusterClass=%s TrainSamplesCount=%u TrainSampleRatio=%f "
"BlockVectorCount=%u",
params.get_as_string(PARAM_IVF_BUILDER_CONVERTER_CLASS).c_str(),
params.get_as_string(PARAM_IVF_BUILDER_QUANTIZER_CLASS).c_str(),
params.get_as_string(PARAM_IVF_BUILDER_OPTIMIZER_CLASS).c_str(),
params.get_as_string(PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_CLASS).c_str(),
params.get_as_bool(PARAM_IVF_BUILDER_QUANTIZE_BY_CENTROID),
params.get_as_bool(PARAM_IVF_BUILDER_STORE_ORIGINAL_FEATURES),
params.get_as_string(PARAM_IVF_BUILDER_CLUSTER_CLASS).c_str(),
params.get_as_uint32(PARAM_IVF_BUILDER_TRAIN_SAMPLE_COUNT),
params.get_as_float(PARAM_IVF_BUILDER_TRAIN_SAMPLE_RATIO),
block_vector_count_);
state_ = INITED;
return 0;
}
int IVFBuilder::cleanup(void) {
LOG_INFO("Begin IVFBuilder::cleanup");
state_ = INIT;
stats_.clear_attributes();
stats_.set_built_costtime(0u);
stats_.set_built_count(0u);
stats_.set_discarded_count(0u);
stats_.set_dumped_costtime(0u);
stats_.set_dumped_count(0u);
stats_.set_trained_costtime(0u);
stats_.set_trained_count(0u);
centroid_num_vec_.clear();
cluster_class_.clear();
converter_class_.clear();
cluster_params_.clear();
labels_.clear();
centroid_index_.reset();
holder_.reset();
converted_meta_ = meta_;
converter_.reset();
quantized_meta_ = meta_;
quantizers_.clear();
error_ = false;
err_code_ = 0;
thread_count_ = 0;
sample_count_ = 0;
cluster_auto_tuning_ = false;
store_original_features_ = false;
quantize_by_centroid_ = false;
LOG_INFO("End IVFBuilder::cleanup");
return 0;
}
int IVFBuilder::train(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) {
LOG_INFO("Begin IVFBuilder::train with holder");
if (state_ != INITED) {
LOG_ERROR("IVFBuilder train failed, wrong state=%d", state_);
return IndexError_Runtime;
}
if (!threads) {
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
if (!threads) {
return IndexError_NoMemory;
}
}
ailego::ElapsedTime timer;
if (!holder || holder->count() == 0) {
LOG_ERROR("Input holder is nullptr or empty while train index");
return IndexError_InvalidArgument;
}
if (!holder->is_matched(meta_)) {
LOG_ERROR("Input holder doesn't match index meta while train index");
return IndexError_Mismatch;
}
if (converter_) {
int ret = IndexConverter::TrainAndTransform(converter_, std::move(holder));
ivf_check_with_msg(ret, "Failed to train or transform by converter %s",
converter_->name().c_str());
converted_meta_ = converter_->meta();
holder = converter_->result();
}
ailego::Params train_params;
int ret = prepare_trainer_params(train_params);
ivf_check_with_msg(ret, "Failed to prepare trainer params, ret=%d", ret);
IndexTrainer::Pointer trainer =
IndexFactory::CreateTrainer("StratifiedClusterTrainer");
ivf_assert_with_msg(trainer, IndexError_NoExist, "Failed to create trainer");
ret = trainer->init(converted_meta_, train_params);
ivf_check_with_msg(ret, "Trainer init failed with ret %d", ret);
ret = trainer->train(std::move(threads), std::move(holder));
ivf_check_with_msg(ret, "Trainer train failed with ret %d", ret);
ret = this->train(trainer);
ivf_check_error_code(ret);
stats_.set_trained_costtime(timer.milli_seconds());
LOG_INFO("End IVFBuilder::train with holder");
state_ = TRAINED;
return 0;
}
int IVFBuilder::train(const IndexTrainer::Pointer &trainer) {
LOG_DEBUG("Begin IVFBuilder::train by trainer");
ailego::ElapsedTime timer;
if (state_ != INITED) {
LOG_ERROR("IVFBuilder train failed, wrong state=%d", state_);
return IndexError_Runtime;
}
if (!trainer) {
LOG_ERROR("Input trainer is nullptr while train index");
return IndexError_InvalidArgument;
}
IndexCluster::CentroidList centroid_list;
IndexBundle::Pointer boundle = trainer->indexes();
int ret = IndexCluster::Deserialize(trainer->meta(), boundle, &centroid_list);
ivf_check_with_msg(ret, "Failed to deserialize index");
const IndexMeta &meta = trainer->meta();
if (meta.data_type() != converted_meta_.data_type() ||
meta.metric_name().compare(converted_meta_.metric_name()) != 0 ||
meta.element_size() != converted_meta_.element_size()) {
if (meta.converter_name() != converter_class_) {
LOG_ERROR("Input trainer doesn't match index meta while train index");
return IndexError_Mismatch;
}
//! Create converter from trainer params
LOG_INFO("Train IVFBuilder by trainer with converter");
converter_ = CreateAndInitConverter(meta_, meta.converter_name(),
meta.converter_params());
ivf_assert(converter_, IndexError_Runtime);
converted_meta_ = meta;
}
centroid_index_ = std::make_shared<IVFCentroidIndex>();
if (!centroid_index_) {
return IndexError_NoMemory;
}
ret = centroid_index_->init(converted_meta_, params_);
ivf_check_error_code(ret);
ret = centroid_index_->build(centroid_list);
ivf_check_with_msg(ret, "Failed to build centroid index");
if (params_.has(PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_CLASS)) {
//! Quantize the centroids for searcher
searcher_centroid_index_ = std::make_shared<IVFCentroidIndex>();
if (!searcher_centroid_index_) {
return IndexError_NoMemory;
}
ailego::Params params;
params_.get(PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_PARAMS, &params);
searcher_centroid_index_->set_quantizer(
params_.get_as_string(PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_CLASS),
params);
ret = searcher_centroid_index_->init(converted_meta_, params_);
ivf_check_error_code(ret);
ret = searcher_centroid_index_->build(centroid_list);
ivf_check_with_msg(ret, "Failed to build centroid index");
}
stats_.set_trained_costtime(timer.milli_seconds());
LOG_DEBUG("End IVFBuilder::train by trainer");
state_ = TRAINED;
return 0;
}
int IVFBuilder::build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) {
LOG_INFO("Begin IVFBuilder::build!");
if (state_ != TRAINED) {
LOG_ERROR("Train the index first before build");
return IndexError_Runtime;
}
ailego::ElapsedTime timer;
if (!holder || holder->count() == 0) {
LOG_ERROR("Input holder is nullptr or empty while building index");
return IndexError_InvalidArgument;
}
if (!holder->is_matched(meta_)) {
LOG_ERROR("Input holder doesn't match index meta while building index");
return IndexError_Mismatch;
}
if (!threads) {
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
if (!threads) {
return IndexError_NoMemory;
}
}
holder_ = std::make_shared<RandomAccessIndexHolder>(meta_);
if (!holder_) {
return IndexError_NoMemory;
}
if (holder->count() > 0) {
holder_->reserve(holder->count());
}
for (auto iter = holder->create_iterator(); iter && iter->is_valid();
iter->next()) {
holder_->emplace(iter->key(), iter->data());
}
// Holder is not needed, cleanup it.
holder.reset();
IndexHolder::Pointer converted_holder = holder_;
if (converter_) {
int ret = converter_->transform(holder_);
ivf_check_with_msg(ret, "Failed to transform by converter %s",
converter_->name().c_str());
converted_holder = converter_->result();
}
labels_.resize(centroid_index_->centroids_count());
int ret = this->build_label_index(threads.get(), converted_holder);
ivf_check_with_msg(ret, "Failed to build index for %s",
IndexError::What(ret));
ret = this->prepare_quantizer(threads.get());
ivf_check_error_code(ret);
stats_.set_built_costtime(timer.milli_seconds());
LOG_INFO("End IVFBuilder::build");
state_ = BUILT;
return 0;
}
int IVFBuilder::dump(const IndexDumper::Pointer &dumper) {
LOG_INFO("Begin IVFBuilder::dump");
if (state_ != BUILT) {
LOG_ERROR("Build the index before dump QC Index");
return IndexError_Runtime;
}
ailego::ElapsedTime timer;
int ret = this->dump_index(dumper);
ivf_check_with_msg(ret, "Failed to dump index with ret=%d", ret);
// the fitting function for the follow points: 1000000(0.02) 10000000(0.01)
// 50000000(0.005) 100000000(0.001)
float scan_ratio = -0.004 * std::log(holder_->count()) + 0.0751;
scan_ratio = std::max(scan_ratio, 0.0001f);
// Set Searcher Params
ailego::Params params;
params.set(PARAM_IVF_SEARCHER_SCAN_RATIO, scan_ratio);
meta_.set_searcher("IVFSearcher", 0, std::move(params));
meta_.set_builder("IVFBuilder", 0, std::move(params_));
ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
if (ret != 0) {
LOG_ERROR("Failed to serialize meta into dumper.");
return ret;
}
stats_.set_discarded_count(stats_.built_count() - stats_.dumped_count());
stats_.set_dumped_costtime(timer.milli_seconds());
LOG_INFO("End IVFBuilder::dump");
return 0;
}
int IVFBuilder::CheckAndUpdateMajorOrder(IndexMeta &meta) {
const std::string &metric_name = meta.metric_name();
auto metric = IndexFactory::CreateMetric(metric_name);
if (!metric) {
LOG_ERROR("CreateMetric %s failed", metric_name.c_str());
return IndexError_InvalidArgument;
}
int ret = metric->init(meta, meta.metric_params());
ivf_check_with_msg(ret, "IndexMetric %s init failed", metric_name.c_str());
bool support_column_major = true;
for (size_t m = 32; m != 0; m /= 2) {
for (size_t n = m; n != 0; n /= 2) {
if (metric->distance_matrix(m, n) == nullptr) {
support_column_major = false;
break;
}
}
if (!support_column_major) {
break;
}
}
support_column_major &=
meta.element_size() % IndexMeta::AlignSizeof(meta.data_type()) == 0;
if (meta.major_order() == IndexMeta::MO_UNDEFINED) {
if (support_column_major && meta.dimension() <= 512) {
meta.set_major_order(IndexMeta::MO_COLUMN);
} else {
meta.set_major_order(IndexMeta::MO_ROW);
}
} else {
if (!support_column_major && meta.major_order() == IndexMeta::MO_COLUMN) {
LOG_WARN(
"Index Metric %s Unsupported "
"Column Major Order",
metric_name.c_str());
return IndexError_Unsupported;
}
}
if (block_vector_count_ * quantized_meta_.element_size() % 32 != 0) {
LOG_ERROR(
"block_vector_count * quantized_element_size not align with 32 bytes.");
return IndexError_InvalidArgument;
}
return 0;
}
int IVFBuilder::parse_centroids_num(const ailego::Params &params) {
std::string centroids_num =
params.get_as_string(PARAM_IVF_BUILDER_CENTROID_COUNT);
if (centroids_num.empty()) {
LOG_ERROR("Param %s is required", PARAM_IVF_BUILDER_CENTROID_COUNT.c_str());
return IndexError_InvalidArgument;
}
std::vector<std::string> centroid_str_vec;
ailego::StringHelper::Split(centroids_num, CENTROID_SEPERATOR,
&centroid_str_vec);
size_t level_cnt = centroid_str_vec.size();
if ((level_cnt <= 0) || (level_cnt > 2)) {
LOG_ERROR("Centroids level count must be [1,2]");
return IndexError_InvalidArgument;
}
for (size_t idx = 0; idx < level_cnt; ++idx) {
uint32_t centroid_cnt = 0;
if (!ailego::StringHelper::ToUint32(centroid_str_vec[idx], &centroid_cnt)) {
LOG_ERROR("Invalid centroids count %s", centroid_str_vec[idx].c_str());
return IndexError_InvalidArgument;
}
centroid_num_vec_.push_back(centroid_cnt);
}
return 0;
}
int IVFBuilder::parse_clustering_params(const ailego::Params &params) {
params.get(PARAM_IVF_BUILDER_CLUSTER_AUTO_TUNING, &cluster_auto_tuning_);
cluster_class_ = params.get_as_string(PARAM_IVF_BUILDER_CLUSTER_CLASS);
if (cluster_class_.empty()) {
// OptKmeansCluster does not support custom metric
cluster_class_ = meta_.metric_name() == kMipsMetricName
? "KmeansCluster"
: "OptKmeansCluster";
LOG_INFO("Using [%s] as default cluster class", cluster_class_.c_str());
}
for (size_t i = 1; i <= centroid_num_vec_.size(); ++i) {
std::string level_params_key =
PARAM_IVF_BUILDER_CLUSTER_PARAMS_IN_LEVEL_PREFIX + std::to_string(i);
ailego::Params level_params;
params.get<ailego::Params>(level_params_key, &level_params);
cluster_params_.push_back(level_params);
}
return 0;
}
int IVFBuilder::parse_general_params(const ailego::Params &params) {
thread_count_ = params.get_as_uint32(PARAM_IVF_BUILDER_THREAD_COUNT);
sample_count_ = params.get_as_uint32(PARAM_IVF_BUILDER_TRAIN_SAMPLE_COUNT);
sample_ratio_ = params.get_as_float(PARAM_IVF_BUILDER_TRAIN_SAMPLE_RATIO);
params.get(PARAM_IVF_BUILDER_QUANTIZE_BY_CENTROID, &quantize_by_centroid_);
params.get(PARAM_IVF_BUILDER_STORE_ORIGINAL_FEATURES,
&store_original_features_);
//! Prepare Converter for training
if (meta_.metric_name() == kIPMetricName) {
converter_class_ = kMipsConverterName;
}
params.get(PARAM_IVF_BUILDER_CONVERTER_CLASS, &converter_class_);
if (!converter_class_.empty()) {
ailego::Params converter_params;
params_.get(PARAM_IVF_BUILDER_CONVERTER_PARAMS, &converter_params);
converter_ =
CreateAndInitConverter(meta_, converter_class_, converter_params);
ivf_assert(converter_, IndexError_NoExist);
}
params_.get(PARAM_IVF_BUILDER_BLOCK_VECTOR_COUNT, &block_vector_count_);
if (block_vector_count_ == 0) {
block_vector_count_ = kDefaultBlockCount;
}
if (block_vector_count_ > kDefaultBlockCount ||
block_vector_count_ & (block_vector_count_ - 1)) {
LOG_ERROR("block_vector_count only can be [1|2|4|8|16|32].");
return IndexError_InvalidArgument;
}
if (block_vector_count_ * meta_.element_size() % 32 != 0) {
LOG_ERROR("block_vector_count * element_size not align with 32 bytes.");
return IndexError_InvalidArgument;
}
return 0;
}
int IVFBuilder::prepare_trainer_params(ailego::Params &params) {
params.set(STRATIFIED_TRAINER_SAMPLE_COUNT, sample_count_);
params.set(STRATIFIED_TRAINER_SAMPLE_RATIO, sample_ratio_);
params.set(STRATIFIED_TRAINER_THREAD_COUNT, thread_count_);
params.set(STRATIFIED_TRAINER_AUTOAUNE, cluster_auto_tuning_);
if (centroid_num_vec_.empty()) {
LOG_ERROR("Centroids no specified.");
return IndexError_InvalidArgument;
}
std::string cluster_count = std::to_string(centroid_num_vec_[0]);
if (centroid_num_vec_.size() > 1) {
cluster_count +=
(CENTROID_SEPERATOR + std::to_string(centroid_num_vec_[1]));
}
params.set(STRATIFIED_TRAINER_CLUSTER_COUNT, cluster_count);
for (size_t i = 1; i <= cluster_params_.size(); ++i) {
std::string level_params_key =
STRATIFIED_TRAINER_PARAMS_IN_LEVEL_PREFIX + std::to_string(i);
params.set(level_params_key, cluster_params_[i - 1]);
}
params.set(STRATIFIED_TRAINER_CLASS_NAME, cluster_class_);
return 0;
}
int IVFBuilder::build_label_index(IndexThreads *threads,
const IndexHolder::Pointer &holder) {
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator for holder failed");
return IndexError_Runtime;
}
auto task_group = threads->make_group();
if (!task_group) {
LOG_ERROR("Failed to create task group");
return IndexError_Runtime;
}
size_t id = 0UL;
AILEGO_DEFER([&]() {
task_group->wait_finish();
stats_.set_built_count(id);
LOG_INFO("Finished building, total=%zu", id);
});
size_t elem_size = holder->element_size();
std::shared_ptr<VectorList> vectors = std::make_shared<VectorList>();
ivf_assert(vectors, IndexError_NoMemory);
for (; iter && iter->is_valid(); iter->next()) {
ivf_assert(!error_, err_code_);
vectors->emplace_back(iter->data(), elem_size, id);
id++;
if (vectors->size() == kBatchSize || id == holder_->count()) {
auto task = ailego::Closure ::New(const_cast<IVFBuilder *>(this),
&IVFBuilder::label, vectors);
task_group->submit(std::move(task));
vectors = std::make_shared<VectorList>();
ivf_assert(vectors, IndexError_NoMemory);
vectors->reserve(kBatchSize);
}
if (!(id & 0xFFFFF)) {
LOG_INFO("Current built count:%zu", id);
}
}
ailego_assert_with(vectors->size() == 0, "invalid size");
return err_code_;
}
int IVFBuilder::dump_index(const IndexDumper::Pointer &dumper) {
int ret = CheckAndUpdateMajorOrder(quantized_meta_);
ivf_check_error_code(ret);
IVFDumper::Pointer ivf_dumper = std::make_shared<IVFDumper>(
quantized_meta_, dumper, centroid_index_->centroids_count(),
block_vector_count_);
if (!ivf_dumper) {
LOG_ERROR("Alloc IVFDumper failed");
return IndexError_NoMemory;
}
//! Dump inverted vectors
std::vector<uint32_t> dumped_ids;
std::function<void(uint32_t)> record_dumped_id = [&](uint32_t) {};
if (store_original_features_) {
dumped_ids.reserve(holder_->count());
record_dumped_id = [&](uint32_t id) { dumped_ids.emplace_back(id); };
}
if (quantizers_.size() == 0) {
//! No quantizer for inverted vectors
for (size_t i = 0; i < centroid_index_->centroids_count(); ++i) {
ailego_assert_with(i < labels_.size(), "Index Overflow");
for (size_t j = 0; j < labels_[i].size(); ++j) {
auto id = labels_[i][j];
record_dumped_id(id);
ret = ivf_dumper->dump_inverted_vector(i, holder_->key(id),
holder_->element(id));
ivf_check_error_code(ret);
}
}
} else {
for (size_t i = 0; i < centroid_index_->centroids_count(); ++i) {
ailego_assert_with(i < labels_.size(), "Index Overflow");
auto holder =
std::make_shared<LabelFilteredIndexHolder>(holder_, labels_[i]);
if (!holder) {
return IndexError_NoMemory;
}
auto quantizer = quantize_by_centroid_ ? quantizers_[i] : quantizers_[0];
ret = quantizer->transform(holder);
ivf_check_error_code(ret);
auto iter = quantizer->result()->create_iterator();
for (; iter->is_valid(); iter->next()) {
uint32_t id = iter->key();
record_dumped_id(id);
ret =
ivf_dumper->dump_inverted_vector(i, holder_->key(id), iter->data());
ivf_check_error_code(ret);
}
}
}
ret = ivf_dumper->dump_inverted_vector_finished();
ivf_check_error_code(ret);
ret = ivf_dumper->dump_quantizer_params(quantizers_);
ivf_check_error_code(ret);
auto centroid_index =
searcher_centroid_index_ ? searcher_centroid_index_ : centroid_index_;
ret = ivf_dumper->dump_centroid_index(centroid_index->data(),
centroid_index->size());
ivf_check_with_msg(ret, "Failed to dump CentroidIndex");
if (store_original_features_) {
for (size_t i = 0; i < dumped_ids.size(); ++i) {
ret = ivf_dumper->dump_original_vector(holder_->element(dumped_ids[i]),
holder_->element_size());
ivf_check_error_code(ret);
}
}
stats_.set_dumped_count(stats_.dumped_count() + ivf_dumper->dumped_count());
return 0;
}
int IVFBuilder::prepare_quantizer(IndexThreads *threads) {
std::string quantizer_name;
params_.get(PARAM_IVF_BUILDER_QUANTIZER_CLASS, &quantizer_name);
if (quantizer_name.empty()) {
return 0;
}
//! Prepare Quantizers for inverted index
ailego::Params quantizer_params;
params_.get(PARAM_IVF_BUILDER_QUANTIZER_PARAMS, &quantizer_params);
if (((quantizer_name != kInt8QuantizerName &&
quantizer_name != kInt4QuantizerName) ||
meta_.metric_name() != kIPMetricName) &&
quantize_by_centroid_) {
LOG_WARN("%s is supported in InnerProduct only",
PARAM_IVF_BUILDER_QUANTIZE_BY_CENTROID.c_str());
quantize_by_centroid_ = false;
}
if (quantizer_name == kInt4QuantizerName && meta_.dimension() & 0x1) {
LOG_ERROR("Unsupport quantizer=%s for dim=%u", kInt4QuantizerName,
meta_.dimension());
return IndexError_Unsupported;
}
int ret = 0;
auto create_and_init_quantizer = [&]() {
auto quantizer = IndexFactory::CreateConverter(quantizer_name);
if (!quantizer) {
LOG_ERROR("Failed to create converter %s", quantizer_name.c_str());
ret = IndexError_NoExist;
return IndexConverter::Pointer();
}
ret = quantizer->init(meta_, quantizer_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize converter %s for %s",
quantizer_name.c_str(), IndexError::What(ret));
return IndexConverter::Pointer();
}
return quantizer;
};
for (size_t i = 0; i < centroid_index_->centroids_count(); ++i) {
quantizers_.emplace_back(create_and_init_quantizer());
ivf_check_error_code(ret);
if (!quantize_by_centroid_) {
break;
}
}
//! Train the quantizers
auto train_data = [&](size_t i) {
IndexHolder::Pointer holder = holder_;
size_t idx = 0;
if (quantize_by_centroid_) {
holder = std::make_shared<LabelFilteredIndexHolder>(holder_, labels_[i]);
if (!holder && !error_.exchange(true)) {
err_code_ = IndexError_NoMemory;
return;
}
idx = i;
}
if (holder->count() == 0) {
return;
}
ret = quantizers_[idx]->train(holder);
if (ret != 0) {
LOG_ERROR("Failed to train converter %s for %s", quantizer_name.c_str(),
IndexError::What(ret));
if (!error_.exchange(true)) {
err_code_ = IndexError_Runtime;
}
}
};
auto task_group = threads->make_group();
if (!task_group) {
LOG_ERROR("Failed to create task group");
return IndexError_Runtime;
}
for (size_t i = 0; i < quantizers_.size(); ++i) {
if (error_) {
task_group->wait_finish();
return err_code_;
}
task_group->submit(ailego::Closure ::New(train_data, i));
}
task_group->wait_finish();
if (quantizers_.size() > 0) {
quantized_meta_ = quantizers_[0]->meta();
}
return 0;
}
INDEX_FACTORY_REGISTER_BUILDER(IVFBuilder);
} // namespace core
} // namespace zvec
+316
View File
@@ -0,0 +1,316 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "framework/index_builder.h"
#include "framework/index_meta.h"
#include "ivf_centroid_index.h"
namespace zvec {
namespace core {
/*! IVF Builder
*/
class IVFBuilder : public IndexBuilder {
public:
//! Constructor
IVFBuilder();
//! Destructor
~IVFBuilder();
//! Disable them
IVFBuilder(const IVFBuilder &) = delete;
IVFBuilder &operator=(const IVFBuilder &) = delete;
public:
//! Initialize the builder
virtual int init(const IndexMeta &meta,
const ailego::Params &params) override;
//! Cleanup the builder
virtual int cleanup(void) override;
//! Train the data
virtual int train(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
//! Train the data
virtual int train(const IndexTrainer::Pointer &trainer) override;
//! Build the index
virtual int build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
//! Dump index into file system
virtual int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
return stats_;
}
IVFCentroidIndex::Pointer centroid_index() const {
return centroid_index_;
}
public:
/*! Random Access Index Holder
*/
class RandomAccessIndexHolder : public IndexHolder {
public:
//! Index Holder Iterator Pointer
typedef std::shared_ptr<RandomAccessIndexHolder> Pointer;
/*! Random Access Index Holder Iterator
*/
class Iterator : public IndexHolder::Iterator {
public:
//! Index Holder Iterator Pointer
typedef std::unique_ptr<Iterator> Pointer;
//! Constructor
Iterator(RandomAccessIndexHolder *owner) : holder_(owner) {}
//! Destructor
virtual ~Iterator(void) {}
//! Retrieve pointer of data
virtual const void *data(void) const override {
return holder_->element(id_);
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
return id_ < holder_->count();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
return holder_->key(id_);
}
//! Next iterator
virtual void next(void) override {
++id_;
}
private:
//! Members
RandomAccessIndexHolder *holder_{nullptr};
uint32_t id_{0};
};
//! Constructor
RandomAccessIndexHolder(const IndexMeta &meta)
: features_(std::make_shared<CompactIndexFeatures>(meta)) {}
//! Retrieve count of elements in holder (-1 indicates unknown)
virtual size_t count(void) const override {
return features_->count();
}
//! Retrieve dimension
virtual size_t dimension(void) const override {
return features_->dimension();
}
//! Retrieve type information
virtual IndexMeta::DataType data_type(void) const override {
return features_->data_type();
}
//! Retrieve element size in bytes
virtual size_t element_size(void) const override {
return features_->element_size();
}
//! Retrieve if it can multi-pass
virtual bool multipass(void) const override {
return true;
}
//! Create a new iterator
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
return IndexHolder::Iterator::Pointer(
new RandomAccessIndexHolder::Iterator(this));
}
void reserve(size_t elems) {
features_->reserve(elems);
keys_.reserve(elems);
}
//! Append an element into holder
void emplace(uint64_t pkey, const void *vec) {
features_->emplace(vec);
keys_.emplace_back(pkey);
}
//! Retrieve feature via local id
const void *element(size_t id) const {
return features_->element(id);
}
//! Retrieve key via local id
uint64_t key(size_t id) const {
ailego_assert_with(id < keys_.size(), "Index Overflow");
return keys_[id];
}
private:
//! Disable them
RandomAccessIndexHolder(void) = delete;
//! Members
CompactIndexFeatures::Pointer features_{};
std::vector<uint64_t> keys_{};
};
private:
/*! Wrapper of feature
*/
class Vector {
public:
typedef std::shared_ptr<Vector> Pointer;
Vector(const void *vec, size_t len, uint32_t idx)
: vec_(reinterpret_cast<const char *>(vec), len), id_{idx} {}
const void *data() const {
return vec_.data();
}
size_t size() const {
return vec_.size();
}
uint32_t id(void) const {
return id_;
}
private:
std::string vec_{};
uint32_t id_{0u};
};
using VectorList = std::vector<Vector>;
//! Check MajorOrder in meta, and update the major order if needed
int CheckAndUpdateMajorOrder(IndexMeta &meta);
//! Parse params
int parse_centroids_num(const ailego::Params &params);
int parse_clustering_params(const ailego::Params &params);
int parse_general_params(const ailego::Params &params);
//! Prepare params for trainer
int prepare_trainer_params(ailego::Params &params);
//! Build the index
int build_label_index(IndexThreads *threads,
const IndexHolder::Pointer &holder);
//! Dump the index to dumper
int dump_index(const IndexDumper::Pointer &dumper);
//! Prepare the quantizer for inverted index
int prepare_quantizer(IndexThreads *threads);
//! Quantize the centrods list
int quantize_centroids();
//! Create converter and init with params
static IndexConverter::Pointer CreateAndInitConverter(
const IndexMeta &meta, const std::string &name,
const ailego::Params &params) {
auto converter = IndexFactory::CreateConverter(name);
if (!converter) {
LOG_ERROR("Failed to create converter %s", name.c_str());
return IndexConverter::Pointer();
}
int ret = converter->init(meta, params);
if (ret != 0) {
LOG_ERROR("Failed to initialize converter %s for %s", name.c_str(),
IndexError::What(ret));
return IndexConverter::Pointer();
}
return converter;
}
//! Select the nearest centroid id for the vector
void label(const std::shared_ptr<VectorList> &vecs) {
for (size_t i = 0; i < vecs->size(); ++i) {
auto &vec = (*vecs)[i];
uint32_t centroid_idx =
centroid_index_->search_nearest_centroid(vec.data(), vec.size());
if (centroid_idx == IVFCentroidIndex::kInvalidID) {
LOG_ERROR("Failed to search nearest centroid in CentroidIndex");
if (!error_.exchange(true)) {
err_code_ = IndexError_Runtime;
}
return;
}
ailego_assert_with(centroid_idx < labels_.size(), "Index Overflow");
mutex_.lock();
labels_[centroid_idx].emplace_back(vec.id());
mutex_.unlock();
}
}
private:
//! Constants
static constexpr size_t kThreadPoolQueueSize = 300u;
static constexpr size_t kBatchSize = 10u;
static constexpr size_t kDefaultBlockCount = 32u;
enum BuilderState { INIT = 0, INITED = 1, TRAINED = 2, BUILT = 3 };
//! Members
BuilderState state_{INIT};
Stats stats_{};
ailego::Params params_{};
IndexMeta meta_{};
std::vector<uint32_t> centroid_num_vec_{};
std::string cluster_class_{};
std::string converter_class_{};
std::vector<ailego::Params> cluster_params_{};
std::vector<std::vector<uint32_t>> labels_{};
std::mutex mutex_{};
IVFCentroidIndex::Pointer centroid_index_{};
IVFCentroidIndex::Pointer searcher_centroid_index_{};
RandomAccessIndexHolder::Pointer holder_{};
IndexMeta converted_meta_{};
IndexConverter::Pointer converter_{};
IndexMeta quantized_meta_{};
std::vector<IndexConverter::Pointer> quantizers_{};
std::atomic_bool error_{false};
int err_code_{0};
uint32_t thread_count_{0};
uint32_t sample_count_{0};
float sample_ratio_{0.0};
uint32_t block_vector_count_{kDefaultBlockCount};
bool cluster_auto_tuning_{false};
bool store_original_features_{false};
bool quantize_by_centroid_{false};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,621 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ivf_centroid_index.h"
#include "framework/index_framework.h"
#include "metric/metric_params.h"
#include "quantizer/quantizer_params.h"
namespace zvec {
namespace core {
/*! Fake Trainer to supply centroids in bundle
*/
class FakeClusterTrainer : public IndexTrainer {
public:
//! Constructor
FakeClusterTrainer(const IndexMeta &imeta, const IndexBundle::Pointer &bundle)
: meta_(imeta), bundle_(bundle) {}
//! Destructor
~FakeClusterTrainer(void) {}
protected:
//! Initialize Trainer
virtual int init(const IndexMeta &, const ailego::Params &) override {
return 0;
}
//! Cleanup Trainer
virtual int cleanup(void) override {
return 0;
}
//! Train the data
virtual int train(IndexHolder::Pointer) override {
return 0;
}
//! Train the data
virtual int train(IndexThreads::Pointer, IndexHolder::Pointer) override {
return 0;
}
//! Load index from file path or dir
virtual int load(IndexStorage::Pointer) override {
return 0;
}
//! Dump index into file path or dir
virtual int dump(const IndexDumper::Pointer &) override {
return 0;
}
//! Retrieve Index Meta
virtual const IndexMeta &meta(void) const override {
return meta_;
}
//! Retrieve statistics
virtual const IndexTrainer::Stats &stats(void) const override {
return stats_;
}
//! Retrieve the output indexes
virtual IndexBundle::Pointer indexes(void) const override {
return bundle_;
}
private:
//! Members
IndexMeta meta_{};
Stats stats_{};
IndexBundle::Pointer bundle_{};
};
/*! Int8QuantizerReformer for InnerProduct Measure
*/
class Int8QuantizerReformer4IP : public IndexReformer {
public:
//! Initialize Reformer
virtual int init(const ailego::Params &) override {
return 0;
}
//! Cleanup Reformer
virtual int cleanup(void) override {
return 0;
}
//! Load index from container
virtual int load(IndexStorage::Pointer) override {
return 0;
}
//! Unload index
virtual int unload(void) override {
return 0;
}
//! Transform query
virtual int transform(const void * /*query*/,
const IndexQueryMeta & /*qmeta*/, std::string * /*out*/,
IndexQueryMeta * /*ometa*/) const override {
#if 0
size_t dim = qmeta.dimension();
out->resize(IndexMeta::ElementSizeof(
IndexMeta::DataType::DT_INT8, dim));
ometa->set_meta(IndexMeta::DataType::DT_INT8, dim);
const float *ivec = reinterpret_cast<const float *>(query);
int8_t *ovec = reinterpret_cast<int8_t *>(&(*out)[0]);
float abs_max = 0.0f;
for (size_t i = 0; i < dim; ++i) {
auto abs = std::abs(ivec[i]);
if (abs > abs_max) {
abs_max = abs;
}
}
if (abs_max > 0.0f) {
float scale = 127 / abs_max;
for (size_t i = 0; i < dim; ++i) {
ovec[i] = static_cast<int8_t>(std::round(ivec[i] * scale));
}
} else {
std::fill(ovec, ovec + dim, static_cast<int8_t>(1));
}
return 0;
#else
return IndexError_NotImplemented;
#endif
}
//! Transform queries
virtual int transform(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, std::string *oquery,
IndexQueryMeta *ometa) const override {
size_t dim = qmeta.dimension();
oquery->resize(count *
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT8, dim));
ometa->set_meta(IndexMeta::DataType::DT_INT8, dim);
const float *ivec = reinterpret_cast<const float *>(query);
int8_t *ovec = reinterpret_cast<int8_t *>(&(*oquery)[0]);
for (size_t q = 0; q < count; ++q) {
float abs_max = 0.0f;
const float *in = &ivec[q * dim];
int8_t *out = &ovec[q * dim];
for (size_t i = 0; i < dim; ++i) {
auto abs = std::abs(in[i]);
if (abs > abs_max) {
abs_max = abs;
}
}
if (abs_max > 0.0f) {
float scale = 127 / abs_max;
for (size_t i = 0; i < dim; ++i) {
out[i] = static_cast<int8_t>(std::round(in[i] * scale));
}
} else {
std::fill(out, out + dim, static_cast<int8_t>(1));
}
}
return 0;
}
//! Normalize results
virtual int normalize(const void * /*query*/,
const IndexQueryMeta & /*qmeta*/,
IndexDocumentList & /*result*/) const override {
return 0;
}
};
/*! Int4QuantizerReformer for InnerProduct Metric
*/
class Int4QuantizerReformer4IP : public IndexReformer {
public:
//! Initialize Reformer
virtual int init(const ailego::Params &) override {
return 0;
}
//! Cleanup Reformer
virtual int cleanup(void) override {
return 0;
}
//! Load index from container
virtual int load(IndexStorage::Pointer) override {
return 0;
}
//! Unload index
virtual int unload(void) override {
return 0;
}
//! Transform query
virtual int transform(const void * /*query*/,
const IndexQueryMeta & /*qmeta*/, std::string * /*out*/,
IndexQueryMeta * /*ometa*/) const override {
return IndexError_NotImplemented;
}
//! Transform queries
virtual int transform(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, std::string *oquery,
IndexQueryMeta *ometa) const override {
if (qmeta.dimension() & 0x1) {
LOG_ERROR("Unsuuport dim=%u for transform", qmeta.dimension());
return IndexError_Unsupported;
}
size_t dim = qmeta.dimension();
oquery->resize(count *
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT4, dim));
ometa->set_meta(IndexMeta::DataType::DT_INT4, dim);
const float *ivec = reinterpret_cast<const float *>(query);
uint8_t *ovec = reinterpret_cast<uint8_t *>(&(*oquery)[0]);
for (size_t q = 0; q < count; ++q) {
float abs_max = 0.0f;
float max = -std::numeric_limits<float>::max();
const float *in = &ivec[q * dim];
uint8_t *out = &ovec[q * dim / 2];
for (size_t i = 0; i < dim; ++i) {
float abs = std::abs(in[i]);
abs_max = std::max(abs_max, abs);
max = std::max(max, in[i]);
}
if (abs_max > 0.0f) {
float scale = ((7 * abs_max > 8 * max) ? 8 : 7) / abs_max;
for (size_t i = 0; i < dim; i += 2) {
auto v1 = static_cast<int8_t>(std::round(in[i] * scale));
auto v2 = static_cast<int8_t>(std::round(in[i + 1] * scale));
out[i / 2] = (static_cast<uint8_t>(v1) << 4) |
(static_cast<uint8_t>(v2) & 0xF);
}
} else {
std::fill(out, out + dim / 2, static_cast<uint8_t>(9));
}
}
return 0;
}
//! Normalize results
virtual int normalize(const void * /*query*/,
const IndexQueryMeta & /*qmeta*/,
IndexDocumentList & /*result*/) const override {
return 0;
}
};
int IVFCentroidIndex::init(const IndexMeta &meta,
const ailego::Params &params) {
meta_ = meta;
params.get(PARAM_IVF_BUILDER_OPTIMIZER_CLASS, &builder_class_);
params.get(PARAM_IVF_BUILDER_OPTIMIZER_PARAMS, &builder_params_);
params.get(PARAM_IVF_SEARCHER_OPTIMIZER, &searcher_class_);
params.get(PARAM_IVF_SEARCHER_OPTIMIZER_PARAMS, &searcher_params_);
return 0;
}
int IVFCentroidIndex::search(const void *query, const IndexQueryMeta &qmeta,
size_t count,
IndexSearcher::Context::Pointer &ctx) {
int ret = 0;
if (reformer_) {
std::string buffer;
IndexQueryMeta ometa;
ret = reformer_->transform(query, qmeta, count, &buffer, &ometa);
if (ret != 0) {
LOG_ERROR("Failed to transform querys by reformer");
return ret;
}
ret = searcher_->search_impl(buffer.data(), ometa, count, ctx);
} else {
ret = searcher_->search_impl(query, qmeta, count, ctx);
}
ivf_check_with_msg(ret, "Failed to search in centroid index for %s",
IndexError::What(ret));
return 0;
}
uint32_t IVFCentroidIndex::search_nearest_centroid(const void *query,
size_t len) {
//! Called in building index precedure, so transform the query is needless
if (len != meta_.element_size()) {
LOG_ERROR("Invalid query size actual: %zu, expected: %u", len,
meta_.element_size());
return kInvalidID;
}
thread_local IndexSearcher::Context::Pointer context(
searcher_->create_context());
context->set_topk(1);
IndexQueryMeta qmeta(meta_.data_type(), meta_.dimension());
int ret = searcher_->search_impl(query, qmeta, context);
if (ret != 0 || context->result().empty()) {
LOG_ERROR("Failed to search nearest centroid, with ret %d", ret);
return kInvalidID;
}
return static_cast<uint32_t>(context->result()[0].key());
}
uint32_t IVFCentroidIndex::transform_and_search_nearest_centroid(
const void *record, const IndexQueryMeta &rmeta,
IndexSearcher::Context::Pointer &ctx) const {
int ret = 0;
if (reformer_) {
std::string buffer;
IndexQueryMeta ometa;
ret = reformer_->convert(record, rmeta, &buffer, &ometa);
if (ret != 0) {
LOG_ERROR("Failed to transform querys by reformer");
return kInvalidID;
}
ret = searcher_->search_impl(buffer.data(), ometa, ctx);
} else {
ret = searcher_->search_impl(record, rmeta, ctx);
}
if (ret != 0 || ctx->result().empty()) {
LOG_ERROR("Failed to search in centroid index for %s",
IndexError::What(ret));
return kInvalidID;
}
return static_cast<uint32_t>(ctx->result()[0].key());
}
IndexHolder::Pointer IVFCentroidIndex::quantize_holder(
const IndexHolder::Pointer &holder) {
auto input = holder;
if (meta_.reformer_name() == kMipsReformerName &&
meta_.metric_name() == kL2MetricName &&
(quantizer_class_ == kInt8QuantizerName ||
quantizer_class_ == kInt4QuantizerName)) {
//! Reverse for Mips if do convert by integer quantizer
auto reverse = IndexFactory::CreateConverter(kMipsRevConverterName);
if (!reverse) {
LOG_ERROR("Failed to create converter %s", kMipsRevConverterName);
return nullptr;
}
ailego::Params params;
auto p = meta_.reformer_params();
params.set(MIPS_REVERSE_CONVERTER_M_VALUE,
p.get_as_uint32(MIPS_REFORMER_M_VALUE));
params.set(MIPS_REVERSE_CONVERTER_U_VALUE,
p.get_as_float(MIPS_REFORMER_U_VALUE));
params.set(MIPS_REVERSE_CONVERTER_L2_NORM,
p.get_as_uint32(MIPS_REFORMER_L2_NORM));
params.set(MIPS_REVERSE_CONVERTER_FORCED_SINGLE_FLOAT,
p.get_as_float(MIPS_REFORMER_FORCED_HALF_FLOAT));
int ret = reverse->init(meta_, params);
if (ret != 0) {
LOG_ERROR("Fail to init converter %s", kMipsRevConverterName);
return nullptr;
}
ret = IndexConverter::TrainAndTransform(reverse, holder);
if (ret != 0) {
LOG_ERROR("Fail to transform converter %s", kMipsRevConverterName);
return nullptr;
}
input = reverse->result();
meta_ = reverse->meta();
meta_.set_metric(kIPMetricName, 0, ailego::Params());
meta_.set_reformer("", 0, ailego::Params());
}
auto converter = IndexFactory::CreateConverter(quantizer_class_);
if (!converter) {
LOG_ERROR("Failed to create converter %s", quantizer_class_.c_str());
return nullptr;
}
int ret = converter->init(meta_, quantizer_params_);
if (ret != 0) {
LOG_ERROR("Fail to init converter %s", quantizer_class_.c_str());
return nullptr;
}
ret = IndexConverter::TrainAndTransform(converter, input);
if (ret != 0) {
LOG_ERROR("Fail to tranform converter %s", quantizer_class_.c_str());
return nullptr;
}
meta_ = converter->meta();
return converter->result();
}
int IVFCentroidIndex::build_index(
const IndexCluster::CentroidList &centroid_list,
const IndexDumper::Pointer &dumper) {
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder(builder_class_);
if (!builder) {
LOG_ERROR("Failed to create builder %s", builder_class_.c_str());
return IndexError_NoExist;
}
IndexHolder::Pointer holder =
std::make_shared<CentroidsIndexHolder>(meta_, centroid_list);
if (!holder) {
return IndexError_NoMemory;
}
if (holder->count() == 0) {
LOG_ERROR("No centroids to build");
return IndexError_InvalidArgument;
}
centroids_count_ = holder->count();
//! Set default params if not given
auto count = std::to_string(
static_cast<size_t>(std::ceil(std::sqrt(centroids_count_ / 10.0))));
// if (IsHcBuilder(builder_class_) &&
// !builder_params_.has(hc::PARAM_HC_BUILDER_CENTROID_COUNT)) {
// builder_params_.set(hc::PARAM_HC_BUILDER_CENTROID_COUNT, count);
// } else if (builder_class_ == "GcBuilder" &&
// !builder_params_.has(hc::PARAM_GC_BUILDER_CENTROID_COUNT)) {
// builder_params_.set(hc::PARAM_GC_BUILDER_CENTROID_COUNT, count);
// }
if (!quantizer_class_.empty()) {
holder = this->quantize_holder(holder);
if (!holder) {
return IndexError_Runtime;
}
}
const auto name = builder_class_.c_str();
int ret = builder->init(meta_, builder_params_);
ivf_check_with_msg(ret, "%s init failed, ret=%d", name, ret);
// if (IsHcBuilder(builder_class_) && quantizer_class_.empty()) {
// auto trainer = this->prepare_trainer(centroid_list);
// ret = trainer ? builder->train(trainer) : builder->train(holder);
// } else {
// ret = builder->train(holder);
// }
ret = builder->train(holder);
ivf_check_with_msg(ret, "%s train failed, ret=%d", name, ret);
ret = builder->build(holder);
ivf_check_with_msg(ret, "%s build failed, ret=%d", name, ret);
ret = builder->dump(dumper);
ivf_check_with_msg(ret, "%s dump failed, ret=%d", name, ret);
ret = dumper->close();
ivf_check_error_code(ret);
return 0;
}
int IVFCentroidIndex::build(const IndexCluster::CentroidList &centroid_list) {
index_building_ = true;
//! Build and dump the index
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
if (!dumper) {
LOG_ERROR("Failed to create MemoryDumper");
return IndexError_NoExist;
}
path_ = IVFUtility::GenerateRandomPath(kTempralPathPrefix);
int ret = dumper->create(path_);
if (ret != 0) {
LOG_ERROR("IndexDumper create path %s failed", path_.c_str());
return ret;
}
ret = this->build_index(centroid_list, dumper);
ivf_check_error_code(ret);
auto rope = IndexMemory::Instance()->open(path_);
if (!rope) {
LOG_ERROR("Open memory path %s failed.", path_.c_str());
return ret;
}
if (rope->count() != 1) {
LOG_ERROR("Graph Rope block count not equal with 1.");
return ret;
}
(*rope)[0].read(0, &data_, 0);
size_ = (*rope)[0].size();
//! Load the index
IndexStorage::Pointer container =
IndexFactory::CreateStorage("MemoryReadStorage");
if (!container) {
LOG_ERROR("Failed to create MemoryReadStorage");
return IndexError_NoExist;
}
ret = container->init(ailego::Params());
ivf_check_with_msg(ret, "Failed to initialize MemoryReadStorage for %s",
IndexError::What(ret));
ret = container->open(path_, false);
ivf_check_with_msg(ret, "Failed to load path in MemoryReadStorage for %s",
IndexError::What(ret));
ailego::Params searcher_params;
if (!searcher_class_.empty()) {
searcher_params.set(PARAM_IVF_SEARCHER_OPTIMIZER, searcher_class_);
}
if (!searcher_params_.empty()) {
searcher_params.set(PARAM_IVF_SEARCHER_OPTIMIZER_PARAMS, searcher_params_);
}
ret = this->load(container, searcher_params);
ivf_check_with_msg(ret, "IVFCentroidIndex load failed with %s",
IndexError::What(ret));
return 0;
}
int IVFCentroidIndex::load(const IndexStorage::Pointer &container,
const ailego::Params params) {
if (!container) {
LOG_ERROR("Invalid container");
return IndexError_InvalidArgument;
}
int ret = IndexHelper::DeserializeFromStorage(container.get(), &meta_);
if (ret != 0) {
LOG_ERROR("Failed to deserialize meta from container");
return ret;
}
auto reformer_name = meta_.reformer_name();
if (!reformer_name.empty()) {
LOG_DEBUG("Load CentroidIndex with reformer %s, metric %s",
reformer_name.c_str(), meta_.metric_name().c_str());
if ((reformer_name == kInt8ReformerName ||
reformer_name == kInt4ReformerName) &&
meta_.metric_name() == kIPMetricName) {
if (reformer_name == kInt8ReformerName) {
reformer_ = std::make_shared<Int8QuantizerReformer4IP>();
} else {
reformer_ = std::make_shared<Int4QuantizerReformer4IP>();
}
if (!reformer_) {
return IndexError_NoMemory;
}
} else {
reformer_ = IndexFactory::CreateReformer(reformer_name);
if (!reformer_) {
LOG_ERROR("Failed to create reformer %s", reformer_name.c_str());
return IndexError_NoExist;
}
}
ret = reformer_->init(meta_.reformer_params());
ivf_check_with_msg(ret, "Failed to initialize reformer %s",
reformer_name.c_str());
}
searcher_class_ = meta_.searcher_name();
params.get(PARAM_IVF_SEARCHER_OPTIMIZER, &searcher_class_);
params.get(PARAM_IVF_SEARCHER_OPTIMIZER_PARAMS, &searcher_params_);
searcher_ = IndexFactory::CreateSearcher(searcher_class_);
if (!searcher_) {
LOG_ERROR("Failed to create searcher %s", searcher_class_.c_str());
return IndexError_Runtime;
}
auto searcher_params = meta_.searcher_params();
searcher_params.merge(searcher_params_);
ret = searcher_->init(searcher_params);
ivf_check_with_msg(ret, "Failed to initialize searcher %s",
searcher_class_.c_str());
IndexMetric::Pointer metric;
if (index_building_) {
// The searcher index metric should specified in building process,
// otherwise the query_metric will be used in searching
metric = IndexFactory::CreateMetric(meta_.metric_name());
ivf_assert_with_msg(metric, IndexError_NoExist,
"Failed to create metric %s",
meta_.metric_name().c_str());
ret = metric->init(meta_, meta_.metric_params());
ivf_check_with_msg(ret, "Failed to initialize metric");
}
ret = searcher_->load(container, metric);
ivf_check_with_msg(ret, "Failed to load searcher %s",
searcher_class_.c_str());
return 0;
}
IndexTrainer::Pointer IVFCentroidIndex::prepare_trainer(
const IndexCluster::CentroidList &centroid_list) {
IndexCluster::CentroidList level1_centroids;
bool two_level = false;
for (auto &it : centroid_list) {
auto centroid = it;
if (!centroid.subitems().empty()) {
two_level = true;
}
centroid.mutable_subitems()->clear();
centroid.mutable_similars()->clear();
level1_centroids.emplace_back(centroid);
}
if (!two_level) {
return IndexTrainer::Pointer();
}
IndexBundle::Pointer bundle;
IndexCluster::Serialize(meta_, level1_centroids, &bundle);
return std::make_shared<FakeClusterTrainer>(meta_, bundle);
}
} // namespace core
} // namespace zvec
+247
View File
@@ -0,0 +1,247 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/container/params.h>
#include "framework/index_framework.h"
#include "ivf_params.h"
#include "ivf_utility.h"
namespace zvec {
namespace core {
/*! IVF Centroid Index
*/
class IVFCentroidIndex {
public:
typedef std::shared_ptr<IVFCentroidIndex> Pointer;
//! Constructor
IVFCentroidIndex(void) {}
//! Destructor
~IVFCentroidIndex(void) {
IndexMemory *instance = IndexMemory::Instance();
if (instance) {
if (instance->has(path_)) {
instance->remove(path_);
}
}
}
//! Initialize
int init(const IndexMeta &meta, const ailego::Params &params);
//! Set Quantizer for the index
void set_quantizer(const std::string &quantizer_name,
ailego::Params &quantizer_params) {
quantizer_class_ = quantizer_name;
quantizer_params_ = quantizer_params;
}
//! Retrieve data address of the index
const void *data(void) const {
return data_;
}
//! Retrieve size of the index
size_t size(void) const {
return size_;
}
//! Create searcher context for centroid index
IndexSearcher::Context::Pointer create_context() const {
return searcher_ ? searcher_->create_context() : nullptr;
}
//! Similarity search
int search(const void *query, const IndexQueryMeta &qmeta, size_t count,
IndexSearcher::Context::Pointer &ctx);
//! Search the nearest point, must be called in local thread pool
uint32_t search_nearest_centroid(const void *query, size_t len);
//! Transform Data and Search the nearest point, called while adding record
uint32_t transform_and_search_nearest_centroid(
const void *record, const IndexQueryMeta &rmeta,
IndexSearcher::Context::Pointer &ctx) const;
//! Build Centroid Index From Centroid List
int build(const IndexCluster::CentroidList &centroid_list);
//! Load Centroid Index From container
int load(const IndexStorage::Pointer &container, const ailego::Params params);
//! Retrieve centroid count of the index
size_t centroids_count(void) const {
return centroids_count_;
}
//! Retrieve meta
const IndexMeta &meta() const {
return meta_;
}
//! Retrieve reformer of the index
const IndexReformer::Pointer reformer(void) const {
return reformer_;
}
static constexpr uint32_t kInvalidID = std::numeric_limits<uint32_t>::max();
private:
/*! Centroids IndexHolder
*/
class CentroidsIndexHolder : public IndexHolder {
public:
class Iterator : public IndexHolder::Iterator {
public:
//! Index Holder Iterator Pointer
typedef std::unique_ptr<Iterator> Pointer;
//! Constructor
Iterator(std::vector<const void *> *features) : features_(features) {}
//! Destructor
virtual ~Iterator(void) {}
//! Retrieve pointer of data
virtual const void *data(void) const override {
return (*features_)[id_];
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
return id_ < features_->size();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
return id_;
}
//! Next iterator
virtual void next(void) override {
++id_;
}
private:
//! Members
std::vector<const void *> *features_{nullptr};
uint32_t id_{0};
};
//! Constructor
CentroidsIndexHolder(const IndexMeta &meta,
const IndexCluster::CentroidList &centroid_list)
: dimension_(meta.dimension()),
element_size_(meta.element_size()),
data_type_(meta.data_type()) {
using CentroidList = IndexCluster::CentroidList;
std::function<void(const CentroidList &)> get_leaf_features =
[&](const CentroidList &cents) {
if (cents.empty()) {
return;
}
for (const auto &it : cents) {
if (it.subitems().empty()) {
features_.emplace_back(it.feature());
} else {
get_leaf_features(it.subitems());
}
}
};
get_leaf_features(centroid_list);
}
//! Retrieve count of elements in holder (-1 indicates unknown)
virtual size_t count(void) const override {
return features_.size();
}
//! Retrieve dimension
virtual size_t dimension(void) const override {
return dimension_;
}
//! Retrieve type information
virtual IndexMeta::DataType data_type(void) const override {
return data_type_;
}
//! Retrieve element size in bytes
virtual size_t element_size(void) const override {
return element_size_;
}
//! Retrieve if it can multi-pass
virtual bool multipass(void) const override {
return true;
}
//! Create a new iterator
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
return IndexHolder::Iterator::Pointer(
new CentroidsIndexHolder::Iterator(&features_));
}
private:
//! Members
std::vector<const void *> features_{};
size_t dimension_{0};
size_t element_size_{0};
IndexMeta::DataType data_type_{IndexMeta::DataType::DT_UNDEFINED};
};
int build_index(const IndexCluster::CentroidList &centroid_list,
const IndexDumper::Pointer &dumper);
//! Prepare trainer for clustering index
IndexTrainer::Pointer prepare_trainer(
const IndexCluster::CentroidList &centroid_list);
//! Quantize the centroid vectors in holder
IndexHolder::Pointer quantize_holder(const IndexHolder::Pointer &holder);
private:
//! Constants
constexpr static const char *kDefaultBuilder = "FlatBuilder";
constexpr static const char *kTempralPathPrefix = "IVF";
//! Members
IndexMeta meta_{};
IndexSearcher::Pointer searcher_{};
IndexReformer::Pointer reformer_{};
std::string builder_class_{kDefaultBuilder};
std::string searcher_class_{};
std::string quantizer_class_{};
std::string path_{};
ailego::Params builder_params_{};
ailego::Params searcher_params_{};
ailego::Params quantizer_params_{};
const void *data_{};
size_t size_{};
size_t centroids_count_{0};
bool index_building_{false};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,46 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ivf_distance_calculator.h"
#include <iostream>
namespace zvec {
namespace core {
IVFDistanceCalculator::IVFDistanceCalculator(const IndexMeta &meta,
const IndexMetric::Pointer &metric,
uint32_t block_vec_cnt)
: metric_ptr_(metric), block_vec_cnt_(block_vec_cnt) {
row_distance_ = metric->distance();
distanceXx1_ = metric->distance_matrix(block_vec_cnt, 1);
distances_.resize(33);
for (size_t b = 32; b != 0; b /= 2) {
distances_[b] = metric->distance_matrix(block_vec_cnt, b);
}
element_size_ = meta.element_size();
dimension_ = meta.dimension();
if (meta.major_order() == IndexMeta::MajorOrder::MO_COLUMN) {
column_major_order_ = true;
} else {
column_major_order_ = false;
}
}
IVFDistanceCalculator::~IVFDistanceCalculator() {
row_distance_ = nullptr;
distanceXx1_ = nullptr;
distances_.clear();
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,185 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/utility/time_helper.h>
#include "framework/index_framework.h"
namespace zvec {
namespace core {
class IVFDistanceCalculator {
public:
typedef std::shared_ptr<IVFDistanceCalculator> Pointer;
//! Constructor
IVFDistanceCalculator(const IndexMeta &meta,
const IndexMetric::Pointer &metric,
uint32_t block_vec_cnt);
virtual ~IVFDistanceCalculator();
public:
inline void query_centroids_distance(const void *query, size_t qnum,
const void *feature, size_t fnum,
float *distances);
inline void query_centroids_distance(const void *query, const void *feature,
size_t fnum, float *distances);
inline void query_features_distance(const void *query, const void *feature,
size_t fnum, float *distances);
inline void query_features_distance(const void *query, const void *feature,
bool column_major, size_t fnum,
float *distances);
protected:
//! Row Major Distances -> Online
inline void row_major_distance(const void *query, size_t qnum,
const void *feature, size_t fnum, float *out);
inline void row_major_distance(const void *query, const void *feature,
size_t fnum, float *out);
template <size_t Q>
inline void batch_query_centroids_distance(const void *query,
const void *feature, size_t fnum,
float *distances);
protected:
IndexMetric::Pointer metric_ptr_{};
IndexMetric::MatrixDistance row_distance_{nullptr};
IndexMetric::MatrixDistance distanceXx1_{nullptr};
std::vector<IndexMetric::MatrixDistance> distances_{};
size_t element_size_{0};
size_t dimension_{0};
uint32_t block_vec_cnt_{0};
bool column_major_order_{false};
};
void IVFDistanceCalculator::query_centroids_distance(const void *query,
size_t qnum,
const void *feature,
size_t fnum,
float *distances) {
if (column_major_order_) {
switch (qnum) {
case 1:
batch_query_centroids_distance<1>(query, feature, fnum, distances);
break;
case 16:
batch_query_centroids_distance<16>(query, feature, fnum, distances);
break;
case 8:
batch_query_centroids_distance<8>(query, feature, fnum, distances);
break;
case 4:
batch_query_centroids_distance<4>(query, feature, fnum, distances);
break;
case 2:
batch_query_centroids_distance<2>(query, feature, fnum, distances);
break;
case 32:
batch_query_centroids_distance<32>(query, feature, fnum, distances);
break;
default:
LOG_ERROR("Unsupported query num %zu.", qnum);
break;
}
} else {
const uint8_t *cur_query = reinterpret_cast<const uint8_t *>(query);
for (size_t q = 0; q < qnum; ++q) {
this->row_major_distance(cur_query, feature, fnum, distances);
cur_query += element_size_;
distances += block_vec_cnt_;
}
}
}
void IVFDistanceCalculator::query_centroids_distance(const void *query,
const void *feature,
size_t fnum,
float *distances) {
this->query_features_distance(query, feature, fnum, distances);
}
void IVFDistanceCalculator::query_features_distance(const void *query,
const void *feature,
size_t fnum,
float *distances) {
if (column_major_order_) {
if (fnum == block_vec_cnt_) {
distanceXx1_(feature, query, dimension_, distances);
} else {
this->row_major_distance(query, feature, fnum, distances);
}
} else {
this->row_major_distance(query, feature, fnum, distances);
}
}
void IVFDistanceCalculator::query_features_distance(const void *query,
const void *feature,
bool column_major,
size_t fnum,
float *distances) {
if (column_major) {
ailego_assert_with(fnum == block_vec_cnt_, "Invalid Block");
distanceXx1_(feature, query, dimension_, distances);
} else {
this->row_major_distance(query, feature, fnum, distances);
}
}
template <size_t Q>
void IVFDistanceCalculator::batch_query_centroids_distance(const void *query,
const void *feature,
size_t fnum,
float *distances) {
if (fnum == block_vec_cnt_) {
distances_[Q](feature, query, dimension_, distances);
} else {
row_major_distance(query, Q, feature, fnum, distances);
}
}
void IVFDistanceCalculator::row_major_distance(const void *query, size_t qnum,
const void *feature, size_t fnum,
float *out) {
const uint8_t *cur_query = reinterpret_cast<const uint8_t *>(query);
for (size_t q = 0; q < qnum; ++q) {
const uint8_t *tmp_feature = reinterpret_cast<const uint8_t *>(feature);
float *cur_out = out + q * fnum;
for (size_t f = 0; f < fnum; ++f) {
row_distance_(cur_query, tmp_feature, dimension_, cur_out + f);
tmp_feature += element_size_;
}
cur_query += element_size_;
}
}
void IVFDistanceCalculator::row_major_distance(const void *query,
const void *feature, size_t fnum,
float *out) {
const uint8_t *cur_feature = reinterpret_cast<const uint8_t *>(feature);
for (size_t f = 0; f < fnum; ++f) {
row_distance_(query, cur_feature, dimension_, out + f);
cur_feature += element_size_;
}
}
} // namespace core
} // namespace zvec
+409
View File
@@ -0,0 +1,409 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ivf_dumper.h"
namespace zvec {
namespace core {
int IVFDumper::dump_inverted_vector(uint32_t inverted_list_id, uint64_t key,
const void *vec) {
int ret = this->check_dump_inverted_list(inverted_list_id);
ivf_check_error_code(ret);
++inverted_lists_meta_[cur_list_id_].vector_count;
++header_.total_vector_count;
block_.emplace(key, vec, IndexMeta::MajorOrder::MO_ROW);
if (block_.full()) {
ret = this->dump_block();
ivf_check_error_code(ret);
}
return 0;
}
int IVFDumper::dump_inverted_block(uint32_t inverted_list_id,
const uint64_t *keys, const void *vecs,
uint32_t vector_count, bool column_major) {
int ret = this->check_dump_inverted_list(inverted_list_id);
ivf_check_error_code(ret);
if (block_.match_order(column_major ? IndexMeta::MajorOrder::MO_COLUMN
: IndexMeta::MajorOrder::MO_ROW) &&
vector_count == block_.capacity()) {
// Dump the block directly
size_t size = vector_count * meta_.element_size();
size_t pd_size = ailego_align(size, 32) - size;
if (dumper_->write(vecs, size) != size) {
LOG_ERROR("Failed to write data into dumper %s", dumper_->name().c_str());
return IndexError_WriteData;
}
if (pd_size > 0) {
std::string padding(pd_size, '\0');
if (dumper_->write(padding.data(), pd_size) != pd_size) {
return IndexError_WriteData;
}
}
std::copy(keys, keys + vector_count, std::back_inserter(keys_));
++inverted_lists_meta_[cur_list_id_].block_count;
++header_.block_count;
header_.inverted_body_size += size;
} else {
size_t step_size = meta_.element_size();
if (column_major) {
step_size = IndexMeta::AlignSizeof(meta_.data_type());
}
for (size_t i = 0; i < vector_count; ++i) {
auto v = reinterpret_cast<const char *>(vecs) + i * step_size;
block_.emplace(keys[i], v,
column_major ? IndexMeta::MajorOrder::MO_COLUMN
: IndexMeta::MajorOrder::MO_ROW);
if (block_.full()) {
ret = this->dump_block();
ivf_check_error_code(ret);
}
}
}
inverted_lists_meta_[cur_list_id_].vector_count += vector_count;
header_.total_vector_count += vector_count;
return 0;
}
int IVFDumper::dump_container_segment(const IndexStorage::Pointer &container,
const std::string &segmemt_id) {
auto seg = container->get(segmemt_id, 2);
if (!seg) {
LOG_ERROR("Failed to fetch segment %s from %s", segmemt_id.c_str(),
container->name().c_str());
return IndexError_InvalidFormat;
}
const size_t batch_size = 32 * 1024;
const size_t total_size = seg->data_size() + seg->padding_size();
size_t off = 0;
while (off < total_size) {
const void *data = nullptr;
size_t rd_size = std::min(batch_size, total_size - off);
if (seg->read(off, &data, rd_size) != rd_size) {
LOG_ERROR("Failed to read data, off=%zu size=%zu", off, rd_size);
return IndexError_ReadData;
}
if (dumper_->write(data, rd_size) != rd_size) {
LOG_ERROR("Failed to write data, size=%zu", rd_size);
return IndexError_WriteData;
}
off += rd_size;
}
int ret = dumper_->append(segmemt_id, seg->data_size(), seg->padding_size(),
seg->data_crc());
ivf_check_with_msg(ret, "Failed to append %s", segmemt_id.c_str());
dumped_size_ += total_size;
return 0;
}
int IVFDumper::dump_inverted_vector_finished(void) {
//! Dump Inverted Index Segment
if (!block_.empty()) {
int ret = this->dump_block();
ivf_check_error_code(ret);
}
header_.block_size = block_.block_size();
size_t segment_size = header_.inverted_body_size;
int ret = dumper_->append(IVF_INVERTED_BODY_SEG_ID, segment_size, 0, 0);
if (ret != 0) {
LOG_ERROR("Failed to append to segment %s, ret=%d",
IVF_INVERTED_BODY_SEG_ID.c_str(), ret);
return ret;
}
dumped_size_ += segment_size;
//! Dump Inverted Index Header Segment
std::string str;
meta_.serialize(&str);
header_.header_size = sizeof(header_) + str.size();
header_.index_meta_size = str.size();
header_.inverted_list_count = inverted_lists_meta_.size();
if (dumper_->write(&header_, sizeof(header_)) != sizeof(header_)) {
LOG_ERROR("Failed to write data, size %zu", sizeof(header_));
return IndexError_WriteData;
}
if (dumper_->write(str.data(), str.size()) != str.size()) {
LOG_ERROR("Failed to write data, size %zu", str.size());
return IndexError_WriteData;
}
size_t padding_size = 0;
ret = this->dump_padding(header_.header_size, &padding_size);
ivf_check_error_code(ret);
ret = dumper_->append(IVF_INVERTED_HEADER_SEG_ID, header_.header_size,
padding_size, 0);
if (ret != 0) {
LOG_ERROR("Failed to append to segment %s, ret:%d",
IVF_INVERTED_HEADER_SEG_ID.c_str(), ret);
return ret;
}
dumped_size_ += header_.header_size + padding_size;
LOG_DEBUG(
"Dump header info: blocks=%u block_size=%u block_vec_count=%u "
"inverted_list_count=%u total_vecs=%u inverted_size=%zu",
header_.block_count, header_.block_size, header_.block_vector_count,
header_.inverted_list_count, header_.total_vector_count,
static_cast<size_t>(header_.inverted_body_size));
//! Dump Inverted Lists Meta Segment
segment_size = inverted_lists_meta_.size() * sizeof(InvertedListMeta);
ret = this->dump_segment(IVF_INVERTED_META_SEG_ID,
inverted_lists_meta_.data(), segment_size);
ivf_check_error_code(ret);
//! Dump Keys Segment
ret = this->dump_segment(IVF_KEYS_SEG_ID, keys_.data(),
keys_.size() * sizeof(keys_[0]));
ivf_check_error_code(ret);
//! Dump Mapping Segment
auto mapping = std::make_shared<std::vector<uint32_t>>();
IVFUtility::Sort(keys_.data(), mapping.get(), keys_.size());
ret = this->dump_segment(IVF_MAPPING_SEG_ID, mapping->data(),
mapping->size() * sizeof(uint32_t));
ivf_check_error_code(ret);
mapping.reset();
//! Dump the Offsets Segment
return this->dump_offsets_segment();
}
int IVFDumper::dump_centroid_index(const void *data, size_t size) {
int ret = this->dump_segment(IVF_CENTROID_SEG_ID, data, size);
ivf_check_error_code(ret);
return 0;
}
int IVFDumper::dump_quantizer_params(
const std::vector<IndexConverter::Pointer> &quantizers) {
if (meta_.reformer_name() != kInt8ReformerName &&
meta_.reformer_name() != kInt4ReformerName) {
// IntegerQuantizer params is support only
return 0;
}
if (quantizers.size() == 1) {
//! Donot dump, using reformer params in IndexMeta
return 0;
}
if (quantizers.size() != header_.inverted_list_count) {
LOG_ERROR("Mismatch size, quantizers=%zu, inverted_list_count=%u",
quantizers.size(), header_.inverted_list_count);
return IndexError_Logic;
}
bool int8_quantizer = meta_.reformer_name() == kInt8ReformerName;
std::vector<InvertedIntegerQuantizerParams> params;
params.resize(header_.inverted_list_count);
for (size_t i = 0; i < quantizers.size(); ++i) {
auto &p = quantizers[i]->meta().reformer_params();
auto &scale_key = int8_quantizer ? INT8_QUANTIZER_REFORMER_SCALE
: INT4_QUANTIZER_REFORMER_SCALE;
auto &bias_key = int8_quantizer ? INT8_QUANTIZER_REFORMER_BIAS
: INT4_QUANTIZER_REFORMER_BIAS;
if (inverted_lists_meta_[i].vector_count > 0 &&
(!p.has(scale_key) || !p.has(bias_key))) {
LOG_ERROR("Miss reformer params %s or %s", bias_key.c_str(),
scale_key.c_str());
return IndexError_Logic;
}
params[i].bias = p.get_as_float(bias_key);
params[i].scale = p.get_as_float(scale_key);
}
return this->dump_segment(
int8_quantizer ? IVF_INT8_QUANTIZED_PARAMS_SEG_ID
: IVF_INT4_QUANTIZED_PARAMS_SEG_ID,
params.data(), params.size() * sizeof(InvertedIntegerQuantizerParams));
}
int IVFDumper::dump_original_vector(const void *data, size_t size) {
if (dumped_feature_count_ >= header_.total_vector_count) {
LOG_ERROR("Dump too much orignal features, expect=%u",
header_.total_vector_count);
return IndexError_Logic;
}
if (dumper_->write(data, size) != size) {
LOG_ERROR("Dumper write features failed");
return IndexError_WriteData;
}
dumped_features_size_ += size;
++dumped_feature_count_;
if (dumped_feature_count_ == header_.total_vector_count) {
//! Dump features finished, dump the meta
size_t padding_size = 0;
int ret = this->dump_padding(size, &padding_size);
ivf_check_error_code(ret);
ret = dumper_->append(IVF_FEATURES_SEG_ID, dumped_features_size_,
padding_size, 0);
if (ret != 0) {
LOG_ERROR("Dumper append segment %s failed, ret:%d",
IVF_FEATURES_SEG_ID.c_str(), ret);
return ret;
}
dumped_size_ += dumped_features_size_;
}
return 0;
}
int IVFDumper::check_dump_inverted_list(uint32_t inverted_list_id) {
if (inverted_list_id < cur_list_id_) {
LOG_ERROR("Invalid backward vector dumping, want=%u cur=%u",
inverted_list_id, cur_list_id_);
return IndexError_Logic;
}
if (inverted_list_id >= inverted_lists_meta_.size()) {
LOG_ERROR("Invalid inverted_list_id=%u, lists_size=%zu", inverted_list_id,
inverted_lists_meta_.size());
return IndexError_Logic;
}
if (inverted_list_id != cur_list_id_) {
//! flush previous inverted_list block
int ret = this->dump_block();
ivf_check_error_code(ret);
for (auto idx = cur_list_id_ + 1; idx <= inverted_list_id; ++idx) {
inverted_lists_meta_[idx].offset = header_.inverted_body_size;
inverted_lists_meta_[idx].id_offset = header_.total_vector_count;
}
cur_list_id_ = inverted_list_id;
}
return 0;
}
int IVFDumper::dump_offsets_segment(void) const {
bool col_pri = meta_.major_order() == IndexMeta::MajorOrder::MO_COLUMN;
size_t total_size = 0;
for (size_t i = 0; i < inverted_lists_meta_.size(); ++i) {
std::vector<InvertedVecLocation> offsets;
const auto &m = inverted_lists_meta_[i];
size_t vec_cnt = m.vector_count;
size_t idx = 0;
uint64_t off = m.offset;
size_t align_idx = vec_cnt - vec_cnt % block_vector_count_;
for (size_t j = 0; j < vec_cnt; ++j) {
if (col_pri && j < align_idx) {
offsets.emplace_back(off + idx * block_.align_size(), true);
} else {
offsets.emplace_back(off + idx * block_.element_size(), false);
}
++idx;
if (idx == block_vector_count_) {
off += header_.block_size;
idx = 0;
}
}
if (idx != 0) {
off += (vec_cnt - align_idx) * meta_.element_size();
}
size_t len = offsets.size() * sizeof(offsets[0]);
size_t actual_len = dumper_->write(offsets.data(), len);
if (actual_len != len) {
LOG_ERROR("Write offsets failed expect %zu, actual: %zu.", len,
actual_len);
return IndexError_WriteData;
}
total_size += len;
}
size_t padding_size = 0;
int ret = this->dump_padding(total_size, &padding_size);
ivf_check_error_code(ret);
ret = dumper_->append(IVF_OFFSETS_SEG_ID, total_size, padding_size, 0);
if (ret != 0) {
LOG_ERROR("Dumper append segment %s failed, ret:%d",
IVF_OFFSETS_SEG_ID.c_str(), ret);
return ret;
}
dumped_size_ += total_size + padding_size;
return 0;
}
int IVFDumper::dump_segment(const std::string &segment_id, const void *data,
size_t size) const {
size_t len = dumper_->write(data, size);
if (len != size) {
LOG_ERROR("Dump segment %s data failed, expect=%zu, actual=%zu",
segment_id.c_str(), size, len);
return IndexError_WriteData;
}
size_t padding_size = 0;
int ret = this->dump_padding(size, &padding_size);
ivf_check_error_code(ret);
uint32_t crc = ailego::Crc32c::Hash(data, size);
ret = dumper_->append(segment_id, size, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
return ret;
}
dumped_size_ += size + padding_size;
return 0;
}
int IVFDumper::dump_padding(size_t data_size, size_t *padding_size) const {
*padding_size = IVFUtility::AlignedSize(data_size) - data_size;
if (*padding_size == 0) {
return 0;
}
std::string padding(*padding_size, '\0');
if (dumper_->write(padding.data(), *padding_size) != *padding_size) {
LOG_ERROR("Append padding failed, size %lu", *padding_size);
return IndexError_WriteData;
}
return 0;
}
int IVFDumper::dump_block(void) {
if (block_.empty()) {
return 0;
}
size_t size = ailego_align(block_.bytes(), 32);
if (dumper_->write(block_.data(), size) != size) {
LOG_ERROR("Failed to write data into dumper %s", dumper_->name().c_str());
return IndexError_WriteData;
}
auto &keys = block_.keys();
std::copy(keys.begin(), keys.end(), std::back_inserter(keys_));
++inverted_lists_meta_[cur_list_id_].block_count;
++header_.block_count;
header_.inverted_body_size += size;
block_.clear();
return 0;
}
} // namespace core
} // namespace zvec
+260
View File
@@ -0,0 +1,260 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "framework/index_framework.h"
#include "metric/metric_params.h"
#include "quantizer/quantizer_params.h"
#include "ivf_index_format.h"
#include "ivf_params.h"
#include "ivf_utility.h"
namespace zvec {
namespace core {
/*! Quantized Clustering Dumper
*/
class IVFDumper {
public:
typedef std::shared_ptr<IVFDumper> Pointer;
//! Vectors block
class Block {
public:
//! Initialize block
void init(const IndexMeta &meta, uint32_t max_vec_count) {
element_size_ = meta.element_size();
auto bsize = IVFUtility::AlignedSize(max_vec_count, element_size_);
data_.resize(bsize);
count_ = 0u;
major_order_ = meta.major_order();
align_size_ = IndexMeta::AlignSizeof(meta.data_type());
units_ = element_size_ / align_size_;
max_vec_count_ = max_vec_count;
keys_.reserve(max_vec_count_);
}
//! Add a vector to the block in row major order
//! If the block is full and the block order is column, make a
//! transpose
void emplace(uint64_t key, const void *vec, IndexMeta::MajorOrder order) {
switch (align_size_) {
case 2:
do_emplace<uint16_t>(vec, order);
break;
case 4:
do_emplace<uint32_t>(vec, order);
break;
case 8:
do_emplace<uint64_t>(vec, order);
break;
default:
ailego_check_with(false, "Unsupport Aligned Size");
}
keys_.emplace_back(key);
}
bool full(void) const {
return count_ == max_vec_count_;
}
const void *data(void) const {
return data_.data();
}
void clear(void) {
count_ = 0u;
keys_.clear();
}
bool empty(void) const {
return count_ == 0u;
}
size_t size(void) const {
return count_;
}
size_t capacity(void) const {
return max_vec_count_;
}
size_t align_size(void) const {
return align_size_;
}
size_t element_size(void) const {
return element_size_;
}
//! Retrieve block data size
size_t bytes(void) const {
return element_size_ * count_;
}
//! Retrieve max block size in bytes
size_t block_size(void) const {
return data_.size();
}
IndexMeta::MajorOrder major_order(void) const {
return major_order_;
}
const std::vector<uint64_t> &keys(void) const {
return keys_;
}
bool match_order(IndexMeta::MajorOrder column_major) const {
return major_order_ == column_major;
}
private:
//! Transpose the block vectors
void transpose() {
std::vector<uint8_t> buf(data_.size());
IVFUtility::Transpose(align_size_, data_.data(), count_, units_,
buf.data());
data_.swap(buf);
}
template <typename T>
void do_emplace(const void *vec, IndexMeta::MajorOrder order) {
ailego_assert_with(count_ < max_vec_count_, "emplace a full block");
T *dst = reinterpret_cast<T *>(data_.data() + element_size_ * count_);
const T *src = reinterpret_cast<const T *>(vec);
size_t step = order == IndexMeta::MO_ROW ? 1 : max_vec_count_;
for (auto i = 0u; i < units_; ++i) {
*dst = *src;
dst++;
src += step;
}
count_++;
if (full() && major_order_ == IndexMeta::MO_COLUMN) {
transpose();
}
}
private:
//! Members
std::vector<uint8_t> data_{};
std::vector<uint64_t> keys_{};
uint32_t count_{0u};
uint32_t units_{0u};
uint32_t align_size_{0u};
uint32_t element_size_{0u};
uint32_t max_vec_count_{0u};
IndexMeta::MajorOrder major_order_{};
};
//! Constructor
IVFDumper(const IndexMeta &meta, const IndexDumper::Pointer &dumper,
size_t inverted_list_count, size_t block_vector_count)
: meta_(meta),
dumper_(dumper),
block_vector_count_(block_vector_count),
inverted_lists_meta_(inverted_list_count) {
block_.init(meta, block_vector_count_);
header_.block_vector_count = block_vector_count_;
}
//! Constructor
IVFDumper(const IndexMeta &meta, const IndexDumper::Pointer &dumper,
size_t inverted_list_count)
: IVFDumper(meta, dumper, inverted_list_count, kDefaultBlockCount) {}
//! Destructor
~IVFDumper() {
// Check the dumper status
if (dumped_feature_count_ > 0 &&
dumped_feature_count_ != header_.total_vector_count) {
LOG_ERROR("Dumped features=%u mismatch from invertedVecs=%u",
dumped_feature_count_, header_.total_vector_count);
ailego_assert_with(false, "invalid status");
}
}
//! Dump a vector in row major order
int dump_inverted_vector(uint32_t inverted_list_id, uint64_t key,
const void *vec);
int dump_inverted_block(uint32_t inverted_list_id, const uint64_t *keys,
const void *vecs, uint32_t vector_count,
bool column_major);
//! Finish dump the inverted vectors
int dump_inverted_vector_finished(void);
//! Dump the centroids index
int dump_centroid_index(const void *data, size_t size);
//! Dump params for each inverted list quantizer
int dump_quantizer_params(
const std::vector<IndexConverter::Pointer> &quantizers);
//! Dump the original vector, which doesnot been quantized
int dump_original_vector(const void *data, size_t size);
//! Retrieve total dumped size
size_t dumped_size(void) const {
return dumped_size_;
}
//! Retrieve total dumped vector count
size_t dumped_count(void) const {
return header_.total_vector_count;
}
//! Dump the segment from container
int dump_container_segment(const IndexStorage::Pointer &container,
const std::string &segmemt_id);
private:
int check_dump_inverted_list(uint32_t inverted_list_id);
//! Dump offsets segment
int dump_offsets_segment(void) const;
//! Dump a segment
int dump_segment(const std::string &segment_id, const void *data,
size_t size) const;
//! Dump segment padding
int dump_padding(size_t data_size, size_t *padding_size) const;
//! Dump a vector block
int dump_block(void);
private:
//! Constants
static constexpr size_t kDefaultBlockCount = 32u;
//! Members
Block block_{}; // vectors grouped in block
const IndexMeta meta_{}; // IndexMeta of the inverted index
const IndexDumper::Pointer dumper_{};
size_t block_vector_count_{kDefaultBlockCount};
std::vector<InvertedListMeta> inverted_lists_meta_{};
std::vector<uint64_t> keys_{};
InvertedIndexHeader header_{};
uint32_t cur_list_id_{0};
uint32_t dumped_feature_count_{0};
size_t dumped_features_size_{0};
mutable size_t dumped_size_{0};
};
} // namespace core
} // namespace zvec
+973
View File
@@ -0,0 +1,973 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ivf_entity.h"
#include <iostream>
#include "ivf_utility.h"
namespace zvec {
namespace core {
//! Initialize
int IVFEntity::IVFReformerWrapper::init(const IndexMeta &imeta) {
auto &name = imeta.reformer_name();
if (name.empty()) {
type_ = kReformerTpNone;
return 0;
}
auto reformer = IndexFactory::CreateReformer(name);
if (!reformer) {
LOG_ERROR("Failed to create reformer %s", name.c_str());
return IndexError_NoExist;
}
int ret = reformer->init(imeta.reformer_params());
ivf_check_with_msg(ret, "Failed to init reformer %s", name.c_str());
reformer_ = std::move(reformer);
if (name == kInt8ReformerName) {
if (imeta.metric_name() == kIPMetricName) {
type_ = kReformerTpInnerProductInt8;
return 0;
}
auto &key = INT8_QUANTIZER_REFORMER_SCALE;
if (!imeta.reformer_params().has(key)) {
LOG_ERROR("Missing param %s in reformer %s", key.c_str(), name.c_str());
return IndexError_InvalidArgument;
};
float scale = imeta.reformer_params().get_as_float(key);
reciprocal_ = scale == 0.0 ? 1.0 : (1.0 / scale);
type_ = kReformerTpInt8;
} else if (name == kInt4ReformerName) {
if (imeta.metric_name() == kIPMetricName) {
type_ = kReformerTpInnerProductInt4;
return 0;
}
auto &key = INT4_QUANTIZER_REFORMER_SCALE;
if (!imeta.reformer_params().has(key)) {
LOG_ERROR("Missing param %s in reformer %s", key.c_str(), name.c_str());
return IndexError_InvalidArgument;
};
float scale = imeta.reformer_params().get_as_float(key);
reciprocal_ = scale == 0.0 ? 1.0 : (1.0 / scale);
type_ = kReformerTpInt4;
} else {
type_ = kReformerTpDefault;
}
LOG_DEBUG("Init QcReformer with %s, type=%u", name.c_str(), type_);
return 0;
}
//! Update the params, Called by gpu searcher only
int IVFEntity::IVFReformerWrapper::update(const IndexMeta &meta) {
auto &name = meta.reformer_name();
if (name == kInt4ReformerName && meta.metric_name() == kL2MetricName) {
auto &key = INT4_QUANTIZER_REFORMER_SCALE;
if (!meta.reformer_params().has(key)) {
LOG_ERROR("Missing param %s in reformer %s", key.c_str(), name.c_str());
return IndexError_InvalidArgument;
};
float scale = meta.reformer_params().get_as_float(key);
reciprocal_ = scale == 0.0 ? 1.0 : (1.0 / scale / kNormalizeScaleFactor);
type_ = kReformerTpInt8;
ailego::Params params;
float int8_scale = scale * kNormalizeScaleFactor;
params.set(INT8_QUANTIZER_REFORMER_SCALE, int8_scale);
float bias =
meta.reformer_params().get_as_float(INT4_QUANTIZER_REFORMER_BIAS);
params.set(INT8_QUANTIZER_REFORMER_BIAS, bias);
params.set(
INT4_QUANTIZER_REFORMER_METRIC,
meta.reformer_params().get_as_string(INT4_QUANTIZER_REFORMER_METRIC));
auto reformer = IndexFactory::CreateReformer(kInt8ReformerName);
if (!reformer) {
LOG_ERROR("Failed to create reformer %s", name.c_str());
return IndexError_NoExist;
}
int ret = reformer->init(params);
ivf_check_with_msg(ret, "Failed to init reformer %s", name.c_str());
reformer_ = reformer;
LOG_DEBUG("Init QcReformer with %s, type=%u", name.c_str(), type_);
}
return 0;
}
//! Transform a query
int IVFEntity::IVFReformerWrapper::transform(const void *query,
const IndexQueryMeta &qmeta,
const void **out,
IndexQueryMeta *ometa) {
int ret = 0;
switch (type_) {
case kReformerTpNone:
*out = query;
*ometa = qmeta;
break;
case kReformerTpInnerProductInt8:
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
return IndexError_Unsupported;
}
scales_.resize(1);
buffer_.resize(IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT8,
qmeta.dimension()));
this->transform(0, static_cast<const float *>(query), qmeta.dimension(),
reinterpret_cast<int8_t *>(&buffer_[0]));
*ometa = qmeta;
ometa->set_meta(IndexMeta::DataType::DT_INT8, qmeta.dimension());
*out = buffer_.data();
break;
case kReformerTpInnerProductInt4:
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
return IndexError_Unsupported;
}
scales_.resize(1);
buffer_.resize(IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT4,
qmeta.dimension()));
this->transform(0, static_cast<const float *>(query), qmeta.dimension(),
reinterpret_cast<uint8_t *>(&buffer_[0]));
*ometa = qmeta;
ometa->set_meta(IndexMeta::DataType::DT_INT4, qmeta.dimension());
*out = buffer_.data();
break;
case kReformerTpInt8:
case kReformerTpInt4:
/* FALLTHRU */
case kReformerTpDefault:
ret = reformer_->transform(query, qmeta, &buffer_, ometa);
*out = buffer_.data();
break;
default:
ret = IndexError_Unsupported;
break;
}
return ret;
}
//! Transform querys
int IVFEntity::IVFReformerWrapper::transform(const void *query,
const IndexQueryMeta &qmeta,
uint32_t count, const void **out,
IndexQueryMeta *ometa) {
int ret = 0;
switch (type_) {
case kReformerTpNone:
*out = query;
*ometa = qmeta;
break;
case kReformerTpInnerProductInt8:
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
return IndexError_Unsupported;
}
scales_.resize(count);
buffer_.resize(count *
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT8,
qmeta.dimension()));
{
const float *ivec = reinterpret_cast<const float *>(query);
int8_t *ovec = reinterpret_cast<int8_t *>(&buffer_[0]);
for (size_t i = 0; i < count; ++i) {
this->transform(i, &ivec[i * qmeta.dimension()], qmeta.dimension(),
&ovec[i * qmeta.dimension()]);
}
}
*ometa = qmeta;
ometa->set_meta(IndexMeta::DataType::DT_INT8, qmeta.dimension());
*out = buffer_.data();
break;
case kReformerTpInnerProductInt4:
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
return IndexError_Unsupported;
}
scales_.resize(count);
buffer_.resize(count *
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT4,
qmeta.dimension()));
{
const float *ivec = reinterpret_cast<const float *>(query);
uint8_t *ovec = reinterpret_cast<uint8_t *>(&buffer_[0]);
for (size_t i = 0; i < count; ++i) {
this->transform(i, &ivec[i * qmeta.dimension()], qmeta.dimension(),
&ovec[i * qmeta.dimension() / 2]);
}
}
*ometa = qmeta;
ometa->set_meta(IndexMeta::DataType::DT_INT4, qmeta.dimension());
*out = buffer_.data();
break;
case kReformerTpInt8:
case kReformerTpInt4:
/* FALLTHRU */
case kReformerTpDefault:
ret = reformer_->transform(query, qmeta, count, &buffer_, ometa);
*out = buffer_.data();
break;
default:
ret = IndexError_Unsupported;
break;
}
return ret;
}
//! Transform querys
int IVFEntity::IVFReformerWrapper::transform_gpu(const void *query,
const IndexQueryMeta &qmeta,
uint32_t count,
const void **out,
IndexQueryMeta *ometa) {
int ret = 0;
switch (type_) {
case kReformerTpNone:
case kReformerTpDefault:
*out = query;
*ometa = qmeta;
break;
case kReformerTpInnerProductInt4:
case kReformerTpInnerProductInt8:
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
return IndexError_Unsupported;
}
scales_.resize(count);
buffer_.resize(count *
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT8,
qmeta.dimension()));
{
const float *ivec = reinterpret_cast<const float *>(query);
int8_t *ovec = reinterpret_cast<int8_t *>(&buffer_[0]);
for (size_t i = 0; i < count; ++i) {
this->transform(i, &ivec[i * qmeta.dimension()], qmeta.dimension(),
&ovec[i * qmeta.dimension()]);
}
}
*ometa = qmeta;
ometa->set_meta(IndexMeta::DataType::DT_INT8, qmeta.dimension());
*out = buffer_.data();
break;
case kReformerTpInt8:
case kReformerTpInt4:
ret = reformer_->transform(query, qmeta, count, &buffer_, ometa);
*out = buffer_.data();
break;
default:
ret = IndexError_Unsupported;
break;
}
return ret;
}
//! Convert a record
int IVFEntity::IVFReformerWrapper::convert(const void *record,
const IndexQueryMeta &rmeta,
const void **out,
IndexQueryMeta *ometa) {
if (type_ == kReformerTpNone) {
*out = record;
*ometa = rmeta;
return 0;
}
int ret = reformer_->convert(record, rmeta, &buffer_, ometa);
*out = buffer_.data();
return ret;
}
//! Convert records
int IVFEntity::IVFReformerWrapper::convert(const void *records,
const IndexQueryMeta &rmeta,
uint32_t count, const void **out,
IndexQueryMeta *ometa) {
if (type_ == kReformerTpNone) {
*out = records;
*ometa = rmeta;
return 0;
}
int ret = reformer_->convert(records, rmeta, count, &buffer_, ometa);
*out = buffer_.data();
return ret;
}
//! Normalize score
void IVFEntity::IVFReformerWrapper::normalize(size_t qidx,
IndexDocumentHeap *heap) const {
switch (type_) {
case kReformerTpNone:
return;
case kReformerTpInnerProductInt8:
case kReformerTpInnerProductInt4:
ailego_assert_with(qidx < scales_.size(), "invalid index");
{
auto reciprocal = 1.0f / scales_[qidx];
for (auto &it : *heap) {
*it.mutable_score() *= reciprocal;
}
}
break;
case kReformerTpInt8:
case kReformerTpInt4:
for (auto &it : *heap) {
*it.mutable_score() *= reciprocal_;
}
break;
default:
// Not support
break;
}
}
//! Normalize score
void IVFEntity::IVFReformerWrapper::normalize(size_t qidx, const void *query,
const IndexQueryMeta &qmeta,
IndexDocumentHeap *heap) const {
switch (type_) {
case kReformerTpNone:
return;
case kReformerTpInnerProductInt8:
case kReformerTpInnerProductInt4:
ailego_assert_with(qidx < scales_.size(), "invalid index");
{
auto reciprocal = 1.0f / scales_[qidx];
for (auto &it : *heap) {
*it.mutable_score() *= reciprocal;
}
}
break;
case kReformerTpInt8:
case kReformerTpInt4:
for (auto &it : *heap) {
*it.mutable_score() *= reciprocal_;
}
break;
case kReformerTpDefault:
reformer_->normalize(query, qmeta, *heap);
break;
default:
// Not support
LOG_ERROR("Not a supported type in QC reformer, type: %u", type_);
break;
}
}
void IVFEntity::IVFReformerWrapper::transform(size_t qidx, const float *in,
size_t dim, int8_t *out) {
ailego_assert_with(qidx < scales_.size(), "invalid index");
float abs_max = 0.0f;
for (size_t i = 0; i < dim; ++i) {
auto abs = std::abs(in[i]);
if (abs > abs_max) {
abs_max = abs;
}
}
if (abs_max > 0.0f) {
float scale = 127 / abs_max;
for (size_t i = 0; i < dim; ++i) {
out[i] = static_cast<int8_t>(std::round(in[i] * scale));
}
scales_[qidx] = scale;
} else {
std::fill(out, out + dim, static_cast<int8_t>(1));
scales_[qidx] = std::numeric_limits<float>::max();
}
}
void IVFEntity::IVFReformerWrapper::transform(size_t qidx, const float *in,
size_t dim, uint8_t *out) {
ailego_assert_with(qidx < scales_.size(), "invalid index");
ailego_assert_with(dim % 2 == 0, "invalid dim");
float abs_max = 0.0f;
float max = -std::numeric_limits<float>::max();
for (size_t i = 0; i < dim; ++i) {
float abs = std::abs(in[i]);
abs_max = std::max(abs_max, abs);
max = std::max(max, in[i]);
}
if (abs_max > 0.0f) {
float scale = ((7 * abs_max > 8 * max) ? 8 : 7) / abs_max;
for (size_t i = 0; i < dim; i += 2) {
auto v1 = static_cast<int8_t>(std::round(in[i] * scale));
auto v2 = static_cast<int8_t>(std::round(in[i + 1] * scale));
out[i / 2] =
(static_cast<uint8_t>(v1) & 0xF) | (static_cast<uint8_t>(v2) << 4);
}
scales_[qidx] = scale;
} else {
std::fill(out, out + dim / 2, static_cast<uint8_t>(9));
scales_[qidx] = std::numeric_limits<float>::max();
}
}
int IVFEntity::load_header(const IndexStorage::Pointer &container) {
//! Load the Header Segment
auto header = container->get(IVF_INVERTED_HEADER_SEG_ID);
if (!header) {
LOG_ERROR("Failed to get segment %s", IVF_INVERTED_HEADER_SEG_ID.c_str());
return IndexError_InvalidFormat;
}
if (header->data_size() < sizeof(header_)) {
LOG_ERROR("Invalid format for segment %s",
IVF_INVERTED_HEADER_SEG_ID.c_str());
return IndexError_InvalidFormat;
}
const void *data = nullptr;
if (header->read(0, &data, header->data_size()) != header->data_size()) {
LOG_ERROR("Failed to read data, segment %s",
IVF_INVERTED_HEADER_SEG_ID.c_str());
return IndexError_ReadData;
}
std::memcpy(&header_, data, sizeof(header_));
if (header_.header_size < sizeof(header_) + header_.index_meta_size ||
header_.header_size > header->data_size()) {
LOG_ERROR("Invalid header size %u", header_.header_size);
return IndexError_InvalidFormat;
}
//! Load the index meta
if (!meta_.deserialize(
reinterpret_cast<const uint8_t *>(data) + sizeof(header_),
header_.index_meta_size)) {
LOG_ERROR("Failed to deserialize index meta");
return IndexError_InvalidFormat;
}
int ret = reformer_.init(meta_);
ivf_check_error_code(ret);
//! Create the distance calculator
auto metric = IndexFactory::CreateMetric(meta_.metric_name());
if (!metric) {
LOG_ERROR("Failed to create metric %s", meta_.metric_name().c_str());
return IndexError_NoExist;
}
ret = metric->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("Failed to initialize metric %s", meta_.metric_name().c_str());
return ret;
}
calculator_ = std::make_shared<IVFDistanceCalculator>(
meta_, metric->query_metric() ? metric->query_metric() : metric,
header_.block_vector_count);
if (!calculator_) {
return IndexError_NoMemory;
}
return 0;
}
int IVFEntity::load(const IndexStorage::Pointer &container) {
int ret = this->load_header(container);
ivf_check_error_code(ret);
//! Load the remaining segments
container_ = container;
size_t expect_size = header_.inverted_body_size;
inverted_ = load_segment(IVF_INVERTED_BODY_SEG_ID, expect_size);
if (!inverted_) {
LOG_ERROR("Failed to load segment, inverted_size=%zu block_count=%u",
static_cast<size_t>(header_.inverted_body_size),
header_.block_count);
return IndexError_InvalidFormat;
}
expect_size = header_.inverted_list_count * sizeof(InvertedListMeta);
inverted_meta_ = load_segment(IVF_INVERTED_META_SEG_ID, expect_size);
if (!inverted_meta_) {
LOG_ERROR("Failed to load segment, inverted_lists=%u",
header_.inverted_list_count);
return IndexError_InvalidFormat;
}
expect_size = header_.total_vector_count * sizeof(uint64_t);
keys_ = load_segment(IVF_KEYS_SEG_ID, expect_size);
if (!keys_) {
return IndexError_InvalidFormat;
}
expect_size = header_.total_vector_count * sizeof(InvertedVecLocation);
offsets_ = load_segment(IVF_OFFSETS_SEG_ID, expect_size);
if (!offsets_) {
return IndexError_InvalidFormat;
}
expect_size = header_.total_vector_count * sizeof(uint32_t);
mapping_ = load_segment(IVF_MAPPING_SEG_ID, expect_size);
if (!mapping_) {
return IndexError_InvalidFormat;
}
norm_value_sqrt_ =
meta_.metric_name() == "Euclidean" || meta_.metric_name() == "Manhattan";
if (container_->get(IVF_INT8_QUANTIZED_PARAMS_SEG_ID) ||
container->get(IVF_INT4_QUANTIZED_PARAMS_SEG_ID)) {
expect_size =
header_.inverted_list_count * sizeof(InvertedIntegerQuantizerParams);
auto &seg_id = meta_.reformer_name() == kInt8ReformerName
? IVF_INT8_QUANTIZED_PARAMS_SEG_ID
: IVF_INT4_QUANTIZED_PARAMS_SEG_ID;
integer_quantizer_params_ = load_segment(seg_id, expect_size);
if (!integer_quantizer_params_) {
return IndexError_InvalidFormat;
}
norm_value_ = 0.0f;
} else if (meta_.reformer_name() == kInt8ReformerName ||
meta_.reformer_name() == kInt4ReformerName) {
auto &scale_key = meta_.reformer_name() == kInt8ReformerName
? INT8_QUANTIZER_REFORMER_SCALE
: INT4_QUANTIZER_REFORMER_SCALE;
auto scale = meta_.reformer_params().get_as_float(scale_key);
norm_value_ = this->convert_to_normalize_value(scale);
} else {
norm_value_ = 1.0f;
}
if (container_->get(IVF_FEATURES_SEG_ID)) {
features_ = load_segment(IVF_FEATURES_SEG_ID, 0);
if (!features_) {
return IndexError_InvalidFormat;
}
if (features_->data_size() % vector_count() != 0) {
LOG_ERROR("Invalid featureSegment size=%zu, totalVecs=%zu",
features_->data_size(), vector_count());
return IndexError_InvalidFormat;
}
}
LOG_DEBUG(
"Load inverted index done, docs=%u invertedListCnt=%u "
"elementSize=%u metric=%s reformer=%s",
header_.total_vector_count, header_.inverted_list_count,
meta_.element_size(), meta_.metric_name().c_str(),
meta_.reformer_name().c_str());
return 0;
}
int IVFEntity::search(size_t inverted_list_id, const void *query,
const IndexFilter &filter, uint32_t *scan_count,
IndexDocumentHeap *heap,
IndexContext::Stats *context_stats) const {
ailego_assert_with(inverted_list_id < header_.inverted_list_count,
"invalid id");
auto list_meta = this->inverted_list_meta(inverted_list_id);
ivf_assert(list_meta, IndexError_ReadData);
const void *data = nullptr;
const size_t block_vecs = header_.block_vector_count;
float distances[block_vecs];
const size_t batch_size = kBatchBlocks;
const size_t block_size = header_.block_size;
const auto norm_val = this->inverted_list_normalize_value(inverted_list_id);
for (size_t i = 0; i < list_meta->block_count; i += batch_size) {
//! Read vecs
const size_t off = list_meta->offset + i * block_size;
const size_t blocks = std::min(batch_size, list_meta->block_count - i);
const size_t size =
std::min(blocks * block_size,
static_cast<size_t>(header_.inverted_body_size - off));
if (inverted_->read(off, &data, size) != size) {
LOG_ERROR("Failed to read block, off=%zu, size=%zu", off, size);
return IndexError_ReadData;
}
//! Read keys
size_t items = std::min(blocks * block_vecs,
list_meta->vector_count - (i * block_vecs));
auto keys = get_keys(list_meta->id_offset + i * block_vecs, items);
if (!keys) {
return IndexError_ReadData;
}
//! Compute distances for each block
for (size_t b = 0; b < blocks; ++b) {
const size_t vecs_count =
std::min(block_vecs, list_meta->vector_count - (i + b) * block_vecs);
auto block_keys = keys + b * block_vecs;
size_t keeps = 0;
ailego_assert_with(block_vecs < sizeof(keeps) * 8, "bits overflow");
for (size_t k = 0; k < vecs_count; ++k) {
if (!filter(block_keys[k])) {
keeps |= (1 << k);
} else {
++(*context_stats->mutable_filtered_count());
}
}
if (keeps == 0) {
continue;
}
const void *block_data = static_cast<const char *>(data) + b * block_size;
calculator_->query_features_distance(query, block_data, vecs_count,
distances);
*(context_stats->mutable_dist_calced_count()) += vecs_count;
uint32_t id_off = list_meta->id_offset + (i + b) * block_vecs;
for (size_t k = 0; k < vecs_count; ++k) {
if (keeps & (1 << k)) {
if (block_keys[k] != kInvalidKey) {
heap->emplace(block_keys[k], distances[k] * norm_val, id_off + k);
}
}
}
}
}
*scan_count = list_meta->vector_count;
return 0;
}
//! search in inverted list without filter
int IVFEntity::search(size_t inverted_list_id, const void *query,
uint32_t *scan_count, IndexDocumentHeap *heap,
IndexContext::Stats *context_stats) const {
ailego_assert_with(inverted_list_id < header_.inverted_list_count,
"invalid id");
auto list_meta = inverted_list_meta(inverted_list_id);
ivf_assert(list_meta, IndexError_ReadData);
const void *data = nullptr;
const size_t block_vecs = header_.block_vector_count;
float distances[block_vecs];
const size_t batch_size = kBatchBlocks;
const size_t block_size = header_.block_size;
const auto norm_val = this->inverted_list_normalize_value(inverted_list_id);
for (size_t i = 0; i < list_meta->block_count; i += batch_size) {
//! Read vecs
const size_t off = list_meta->offset + i * block_size;
const size_t blocks = std::min(batch_size, list_meta->block_count - i);
const size_t size =
std::min(blocks * block_size,
static_cast<size_t>(header_.inverted_body_size - off));
if (inverted_->read(off, &data, size) != size) {
LOG_ERROR("Failed to read block, off=%zu, size=%zu", off, size);
return IndexError_ReadData;
}
//! Read keys
size_t items = std::min(blocks * block_vecs,
list_meta->vector_count - (i * block_vecs));
auto keys = get_keys(list_meta->id_offset + i * block_vecs, items);
if (!keys) {
return IndexError_ReadData;
}
//! Compute distances for each block
for (size_t b = 0; b < blocks; ++b) {
const size_t vecs_count =
std::min(block_vecs, list_meta->vector_count - (i + b) * block_vecs);
auto block_keys = keys + b * block_vecs;
const void *block_data = static_cast<const char *>(data) + b * block_size;
calculator_->query_features_distance(query, block_data, vecs_count,
distances);
for (size_t k = 0; k < vecs_count; ++k) {
if (block_keys[k] != kInvalidKey) {
uint32_t id = list_meta->id_offset + (i + b) * block_vecs + k;
heap->emplace(block_keys[k], distances[k] * norm_val, id);
}
}
*(context_stats->mutable_dist_calced_count()) += vecs_count;
}
}
*scan_count = list_meta->vector_count;
return 0;
}
//! search all inverted list with filter
int IVFEntity::search(const void *query, const IndexFilter &filter,
IndexDocumentHeap *heap,
IndexContext::Stats *context_stats) const {
for (size_t i = 0; i < header_.inverted_list_count; ++i) {
uint32_t scan_count;
int ret = this->search(i, query, filter, &scan_count, heap, context_stats);
if (ret != 0) {
return ret;
}
}
return 0;
}
//! search all inverted list without filter
int IVFEntity::search(const void *query, IndexDocumentHeap *heap,
IndexContext::Stats *context_stats) const {
for (size_t i = 0; i < header_.inverted_list_count; ++i) {
uint32_t scan_count;
int ret = this->search(i, query, &scan_count, heap, context_stats);
if (ret != 0) {
return ret;
}
}
return 0;
}
const void *IVFEntity::get_vector(size_t id) const {
if (features_) {
const void *data = nullptr;
size_t element_size = features_->data_size() / vector_count();
size_t off = id * element_size;
if (features_->read(off, &data, element_size) != element_size) {
LOG_ERROR("Failed to read segment, off=%zu size=%zu", off, element_size);
return nullptr;
}
return data;
}
const void *data = nullptr;
size_t size = sizeof(InvertedVecLocation);
if (offsets_->read(id * size, &data, size) != size) {
LOG_ERROR("Failed to read offsets segment, id=%zu", id);
return nullptr;
}
auto &loc = *reinterpret_cast<const InvertedVecLocation *>(data);
if (loc.column_major) {
vector_.resize(meta_.element_size());
auto unit_size = IndexMeta::AlignSizeof(meta_.data_type());
size_t cols = meta_.element_size() / unit_size;
size_t step = block_vector_count() * unit_size;
size_t rd_size = step * (cols - 1) + unit_size;
if (inverted_->read(loc.offset, &data, rd_size) != rd_size) {
LOG_ERROR("Failed to read data, off=%zu size=%zu",
static_cast<size_t>(loc.offset), rd_size);
return nullptr;
}
for (size_t c = 0; c < cols; ++c) {
vector_.replace(c * unit_size, unit_size,
reinterpret_cast<const char *>(data) + c * step,
unit_size);
}
return vector_.data();
} else {
if (inverted_->read(loc.offset, &data, meta_.element_size()) !=
meta_.element_size()) {
LOG_ERROR("Failed to read data, off=%zu size=%u",
static_cast<size_t>(loc.offset), meta_.element_size());
return nullptr;
}
return data;
}
}
int IVFEntity::get_vector(size_t id, IndexStorage::MemoryBlock &block) const {
if (features_) {
size_t element_size = features_->data_size() / vector_count();
size_t off = id * element_size;
if (features_->read(off, block, element_size) != element_size) {
LOG_ERROR("Failed to read segment, off=%zu size=%zu", off, element_size);
return IndexError_Runtime;
}
return 0;
}
IndexStorage::MemoryBlock data_block;
size_t size = sizeof(InvertedVecLocation);
if (offsets_->read(id * size, data_block, size) != size) {
LOG_ERROR("Failed to read offsets segment, id=%zu", id);
return IndexError_Runtime;
}
const void *data = data_block.data();
auto &loc = *reinterpret_cast<const InvertedVecLocation *>(data);
if (loc.column_major) {
vector_.resize(meta_.element_size());
auto unit_size = IndexMeta::AlignSizeof(meta_.data_type());
size_t cols = meta_.element_size() / unit_size;
size_t step = block_vector_count() * unit_size;
size_t rd_size = step * (cols - 1) + unit_size;
if (inverted_->read(loc.offset, &data, rd_size) != rd_size) {
LOG_ERROR("Failed to read data, off=%zu size=%zu",
static_cast<size_t>(loc.offset), rd_size);
return IndexError_Runtime;
}
for (size_t c = 0; c < cols; ++c) {
vector_.replace(c * unit_size, unit_size,
reinterpret_cast<const char *>(data) + c * step,
unit_size);
}
block.reset(vector_.data());
return 0;
} else {
if (inverted_->read(loc.offset, block, meta_.element_size()) !=
meta_.element_size()) {
LOG_ERROR("Failed to read data, off=%zu size=%u",
static_cast<size_t>(loc.offset), meta_.element_size());
return IndexError_Runtime;
}
return 0;
}
}
uint32_t IVFEntity::key_to_id(uint64_t key) const {
//! Do binary search
uint32_t start = 0UL;
uint32_t end = vector_count();
const void *data = nullptr;
uint32_t idx = 0u;
while (start < end) {
idx = start + (end - start) / 2;
if (ailego_unlikely(mapping_->read(idx * sizeof(uint32_t), &data,
sizeof(uint32_t)) != sizeof(uint32_t))) {
LOG_ERROR("Failed to read mapping segment, idx=%u", idx);
return std::numeric_limits<uint32_t>::max();
}
const uint64_t *mkey;
uint32_t local_id = *reinterpret_cast<const uint32_t *>(data);
if (ailego_unlikely(keys_->read(local_id * sizeof(uint64_t),
(const void **)(&mkey),
sizeof(uint64_t)) != sizeof(uint64_t))) {
LOG_ERROR("Read key from segment failed");
return std::numeric_limits<uint32_t>::max();
}
if (*mkey < key) {
start = idx + 1;
} else if (*mkey > key) {
end = idx;
} else {
return local_id;
}
}
return std::numeric_limits<uint32_t>::max();
}
const void *IVFEntity::get_vector_by_key(uint64_t key) const {
uint32_t id = this->key_to_id(key);
if (id != std::numeric_limits<uint32_t>::max()) {
return get_vector(id);
} else {
return nullptr;
}
}
int IVFEntity::get_vector_by_key(uint64_t key,
IndexStorage::MemoryBlock &block) const {
uint32_t id = this->key_to_id(key);
if (id != std::numeric_limits<uint32_t>::max()) {
return get_vector(id, block);
} else {
return IndexError_Runtime;
}
}
IVFEntity::Pointer IVFEntity::clone(void) const {
auto entity = std::make_shared<IVFEntity>();
return clone(entity);
}
IVFEntity::Pointer IVFEntity::clone(const IVFEntity::Pointer &entity) const {
if (!entity) {
LOG_ERROR("Failed to alloc IVFEntity");
return nullptr;
}
auto inverted = inverted_->clone();
ivf_assert_with_msg(inverted, nullptr, "Failed to clone inverted segment");
auto inverted_meta = inverted_meta_->clone();
ivf_assert_with_msg(inverted_meta, nullptr,
"Failed to clone inverted meta segment");
auto keys = keys_->clone();
ivf_assert_with_msg(keys, nullptr, "Failed to clone keys segment");
auto offsets = offsets_->clone();
ivf_assert_with_msg(offsets, nullptr, "Failed to clone offsets segment");
auto mapping = mapping_->clone();
ivf_assert_with_msg(mapping, nullptr, "Failed to clone mapping segment");
IndexStorage::Segment::Pointer integer_quantizer_params;
if (integer_quantizer_params_) {
integer_quantizer_params = integer_quantizer_params_->clone();
if (!integer_quantizer_params) {
LOG_ERROR("Failed to clone integer quantizer params segment");
return nullptr;
}
}
IndexStorage::Segment::Pointer features;
if (features_) {
features = features_->clone();
if (!features) {
LOG_ERROR("Failed to clone features segment");
return nullptr;
}
}
entity->meta_ = this->meta_;
entity->reformer_ = this->reformer_;
entity->calculator_ = this->calculator_;
entity->header_ = this->header_;
entity->container_ = this->container_;
entity->inverted_ = inverted;
entity->inverted_meta_ = inverted_meta;
entity->keys_ = keys;
entity->offsets_ = offsets;
entity->mapping_ = mapping;
entity->integer_quantizer_params_ = integer_quantizer_params;
entity->features_ = features;
entity->norm_value_ = this->norm_value_;
entity->norm_value_sqrt_ = this->norm_value_sqrt_;
return entity;
}
IndexStorage::Segment::Pointer IVFEntity::load_segment(
const std::string &seg_id, size_t expect_size) const {
auto segment = container_->get(seg_id);
if (!segment) {
LOG_ERROR("Failed to get segment %s", seg_id.c_str());
return nullptr;
}
if (expect_size && segment->data_size() != expect_size) {
LOG_ERROR("Invalid segment %s size=%zu, total_vecs=%u", seg_id.c_str(),
segment->data_size(), header_.total_vector_count);
return nullptr;
}
return segment;
}
} // namespace core
} // namespace zvec
+351
View File
@@ -0,0 +1,351 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "framework/index_framework.h"
#include "metric/metric_params.h"
#include "quantizer/quantizer_params.h"
#include "ivf_distance_calculator.h"
#include "ivf_index_format.h"
#include "ivf_params.h"
namespace zvec {
namespace core {
/*! IVF Entity
*/
class IVFEntity {
public:
typedef std::shared_ptr<IVFEntity> Pointer;
class IVFReformerWrapper;
//! Constructor
IVFEntity() {}
//! Destructor
virtual ~IVFEntity() {}
//! Disable them
IVFEntity(const IVFEntity &) = delete;
IVFEntity &operator=(const IVFEntity &) = delete;
//! load the index from container
virtual int load(const IndexStorage::Pointer &container);
//! search in inverted list with filter
int search(size_t inverted_list_id, const void *query,
const IndexFilter &filter, uint32_t *scan_count,
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const;
//! search in inverted list without filter
int search(size_t inverted_list_id, const void *query, uint32_t *scan_count,
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const;
//! search all inverted list with filter
int search(const void *query, const IndexFilter &filter,
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const;
//! search all inverted list without filter
int search(const void *query, IndexDocumentHeap *heap,
IndexContext::Stats *context_stats) const;
//! Clone the entity
virtual IVFEntity::Pointer clone(void) const;
//! Clone the entity
IVFEntity::Pointer clone(const IVFEntity::Pointer &entity) const;
//! Retrieve the primary keys by local id in heap
int retrieve_keys(IndexDocumentHeap *heap) const {
for (auto &it : (*heap)) {
uint64_t key = this->get_key(it.index());
if (key == kInvalidKey) {
return IndexError_ReadData;
}
it.set_key(key);
}
return 0;
}
//! Retrieve the total vectors in the index
size_t vector_count(void) const {
return header_.total_vector_count;
}
//! Retrieve the inverted list count
size_t inverted_list_count(void) const {
return header_.inverted_list_count;
}
//! Retrieve block size of the inverted vector
size_t inverted_block_size(void) const {
return header_.block_size;
}
//! Retrieve the vectors count in one block
size_t block_vector_count(void) const {
return header_.block_vector_count;
}
//! Retrieve IndexMeta of the inverted index
const IndexMeta &meta(void) const {
return meta_;
}
//! Retrieve a block of vectors
const void *read_block(size_t inverted_list_id, size_t local_block_id,
size_t *vecs_count) const {
auto iv_meta = this->inverted_list_meta(inverted_list_id);
if (!iv_meta || local_block_id >= iv_meta->block_count) {
LOG_ERROR("Failed to read inverted list, listId=%zu blockIdx=%zu",
inverted_list_id, local_block_id);
return nullptr;
}
size_t block_vecs = header_.block_vector_count;
*vecs_count = std::min(block_vecs,
iv_meta->vector_count - local_block_id * block_vecs);
ailego_assert_with(*vecs_count <= header_.block_vector_count,
"invalid vecs");
const size_t off = iv_meta->offset + local_block_id * header_.block_size;
const size_t size = *vecs_count * meta_.element_size();
const void *data = nullptr;
if (inverted_->read(off, &data, size) != size) {
LOG_ERROR("Failed to read block off=%zu size=%zu", off, size);
return nullptr;
}
return data;
}
//! Retrieve the inverted list meta
const InvertedListMeta *inverted_list_meta(size_t inverted_list_id) const {
const void *data = nullptr;
const size_t size = sizeof(InvertedListMeta);
const size_t offset = inverted_list_id * size;
if (inverted_meta_->read(offset, &data, size) != size) {
LOG_ERROR("Failed to read inverted meta, id=%zu, size=%zu",
inverted_list_id, size);
return nullptr;
}
return static_cast<const InvertedListMeta *>(data);
}
//! Retrieve the keys by consecutive local ids
const uint64_t *get_keys(size_t id, size_t count) const {
const void *data = nullptr;
const size_t offset = id * sizeof(uint64_t);
const size_t size = count * sizeof(uint64_t);
if (keys_->read(offset, &data, size) != size) {
LOG_ERROR("Failed to read keys, id=%zu, size=%zu", id, size);
return nullptr;
}
return static_cast<const uint64_t *>(data);
}
//! Retrieve the key by local id
uint64_t get_key(size_t id) const {
const void *data = nullptr;
const size_t offset = id * sizeof(uint64_t);
const size_t size = sizeof(uint64_t);
if (keys_->read(offset, &data, size) != size) {
LOG_ERROR("Failed to read key, id=%zu", id);
return kInvalidKey;
}
return *static_cast<const uint64_t *>(data);
}
//! Retrieve vector by local id
const void *get_vector(size_t id) const;
//! Retrieve vector by local id
const void *get_vector_by_key(uint64_t key) const;
int get_vector(size_t id, IndexStorage::MemoryBlock &block) const;
int get_vector_by_key(uint64_t key, IndexStorage::MemoryBlock &block) const;
uint32_t key_to_id(uint64_t key) const;
//! Transform a query
int transform(const void *query, const IndexQueryMeta &qmeta,
const void **out, IndexQueryMeta *ometa) const {
return reformer_.transform(query, qmeta, out, ometa);
}
//! Transform queries
int transform(const void *query, const IndexQueryMeta &qmeta, uint32_t count,
const void **out, IndexQueryMeta *ometa) const {
return reformer_.transform(query, qmeta, count, out, ometa);
}
//! Normalize the score in query part
void normalize(size_t qidx, IndexDocumentHeap *heap) const {
return reformer_.normalize(qidx, heap);
}
//! Retrieve the value for each inverted list to multiply for normalizing
float inverted_list_normalize_value(size_t inverted_list_id) const {
if (norm_value_ != 0.0f) {
return norm_value_;
}
// ailego_assert_with(integer_quantizer_params_, "nullptr");
if (integer_quantizer_params_ != nullptr) {
const void *data = nullptr;
size_t size = sizeof(InvertedIntegerQuantizerParams);
size_t off = inverted_list_id * size;
if (integer_quantizer_params_->read(off, &data, size) != size) {
LOG_ERROR("Failed to read data from segment, off=%zu", off);
return 1.0f;
}
auto scale =
static_cast<const InvertedIntegerQuantizerParams *>(data)->scale;
return this->convert_to_normalize_value(scale);
}
return norm_value_;
}
//! Check whether the feature segment exist
bool has_orignal_feature() const {
return !!features_;
}
//! Retrieve reformer
const IVFReformerWrapper &reformer(void) const {
return reformer_;
}
/*! Index Reformer Wrapper
* To transform query in inverted index searching, and normalize the score
*/
class IVFReformerWrapper {
public:
//! Constructor
IVFReformerWrapper() {}
//! Assignment
IVFReformerWrapper &operator=(const IVFReformerWrapper &wrapper) {
reformer_ = wrapper.reformer_;
type_ = wrapper.type_;
buffer_.clear();
buffer_.shrink_to_fit();
reciprocal_ = wrapper.reciprocal_;
return *this;
}
//! Initialize
int init(const IndexMeta &imeta);
//! Update
int update(const IndexMeta &meta);
//! Transform a query
int transform(const void *query, const IndexQueryMeta &qmeta,
const void **out, IndexQueryMeta *ometa);
//! Transform queries
int transform(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, const void **out, IndexQueryMeta *ometa);
//! Convert a record
virtual int convert(const void *record, const IndexQueryMeta &rmeta,
const void **out, IndexQueryMeta *ometa);
//! Convert records
virtual int convert(const void *records, const IndexQueryMeta &rmeta,
uint32_t count, const void **out,
IndexQueryMeta *ometa);
//! Transform queries
int transform_gpu(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, const void **out, IndexQueryMeta *ometa);
//! Normalize the score in query part
void normalize(size_t qidx, IndexDocumentHeap *heap) const;
//! Normalize the score in query part
void normalize(size_t qidx, const void *query, const IndexQueryMeta &qmeta,
IndexDocumentHeap *heap) const;
private:
//! Transform query from fp32 to int8
void transform(size_t qidx, const float *in, size_t dim, int8_t *out);
//! Transform query from fp32 to int4
void transform(size_t qidx, const float *in, size_t dim, uint8_t *out);
private:
//! Constants
enum Type {
kReformerTpNone = 0,
kReformerTpInnerProductInt8 = 1,
kReformerTpInnerProductInt4 = 2,
kReformerTpInt8 = 3,
kReformerTpInt4 = 4,
kReformerTpDefault = 7,
};
//! Members
Type type_{kReformerTpNone};
IndexReformer::Pointer reformer_{};
std::string buffer_{};
float reciprocal_{0.0}; // for int8
std::vector<float> scales_{}; // for int8 IP
};
private:
//! Load the segment by seg_id in expect_size segment size
IndexStorage::Segment::Pointer load_segment(const std::string &seg_id,
size_t expect_size) const;
//! Load the header segment
int load_header(const IndexStorage::Pointer &container);
//! Convert the int8 quantizer scale to normalize value
float convert_to_normalize_value(float scale) const {
auto v = scale == 0.0 ? 1.0 : (1.0 / scale);
return !norm_value_sqrt_ ? v : std::sqrt(v);
}
protected:
//! Constants
static constexpr size_t kBatchBlocks = 10u;
//! Members
IndexMeta meta_{};
mutable IVFReformerWrapper reformer_{};
IVFDistanceCalculator::Pointer calculator_{};
InvertedIndexHeader header_{};
IndexStorage::Pointer container_{};
IndexStorage::Segment::Pointer inverted_{};
IndexStorage::Segment::Pointer inverted_meta_{};
IndexStorage::Segment::Pointer keys_{};
IndexStorage::Segment::Pointer offsets_{};
IndexStorage::Segment::Pointer mapping_{};
IndexStorage::Segment::Pointer features_{};
IndexStorage::Segment::Pointer integer_quantizer_params_{};
mutable std::string vector_{}; // temporary buffer for colomn major order
float norm_value_{0.0f}; // normalize the inverted vector to orignal score
bool norm_value_sqrt_{false}; // does the norm value need to sqrt
};
} // namespace core
} // namespace zvec
+167
View File
@@ -0,0 +1,167 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/container/bitmap.h>
#include "framework/index_framework.h"
namespace zvec {
namespace core {
static constexpr uint64_t kInvalidKey = std::numeric_limits<uint64_t>::max();
/*! Index Format of Inverted Index Header
*/
struct InvertedIndexHeader {
uint32_t header_size{0};
uint32_t total_vector_count{0};
uint64_t inverted_body_size{0};
uint32_t inverted_list_count{0};
uint32_t block_vector_count{0};
uint32_t block_size{0};
uint32_t block_count{0};
uint32_t index_meta_size{0};
char reserved_[28];
char index_meta[0];
};
/*! Index Format of Inverted Index Meta for each Inverted list
*/
struct InvertedListMeta {
uint64_t offset{0};
uint32_t block_count{0};
uint32_t vector_count{0};
uint32_t id_offset{0};
char reserved_[16];
};
/*! Index Format of Location in Inverted Index for each vector
*/
struct InvertedVecLocation {
InvertedVecLocation(size_t off, bool col)
: offset(off), column_major(col), reserved(0u) {}
uint64_t offset : 48; // feature offset in posting block segment
uint64_t column_major : 1; // coloum major if true
uint64_t reserved : 15;
};
/*! Index Format of Integer Quantizer params for each inverted list
*/
struct InvertedIntegerQuantizerParams {
float scale{1.0};
float bias{0.0};
};
/*! Location of Vectors Block in Storage Segment
*/
struct BlockLocation {
uint16_t segment_id;
uint16_t block_index;
};
/*! The Header of a Block in Storage Segment
*/
struct BlockHeader {
BlockLocation next;
uint16_t vector_count;
uint16_t column_major : 1;
uint16_t reserved_ : 15;
};
struct DeletionMap {
void set(uint32_t index) {
bitset.set(index);
}
void reset(uint32_t index) {
bitset.reset(index);
}
bool test(uint32_t index) const {
return bitset.test(index);
}
bool is_dirty() const {
return bitset.test_any();
}
ailego::FixedBitset<32> bitset{};
};
static_assert(sizeof(DeletionMap) == 4, "DeletionMap must be 4 bytes");
/*! Meta Information of Streamer Entity
*/
struct StreamerInvertedMeta {
uint64_t create_time{0};
uint64_t update_time{0};
uint64_t revision_id{0};
uint32_t segment_count{0};
uint32_t segment_size{0};
uint8_t reserved_[32];
InvertedIndexHeader header;
};
/*! Location of Vector in Storage Segment
*/
struct VectorLocation {
//! Constructor
VectorLocation(void) {}
//! Constructor
VectorLocation(uint16_t id, bool col, uint32_t off)
: segment_id(id), column_major(col), offset(off) {}
uint16_t segment_id;
uint16_t column_major : 1;
uint16_t reserved_ : 15;
uint32_t offset;
public:
bool operator==(const VectorLocation &other) const {
return segment_id == other.segment_id &&
column_major == other.column_major && offset == other.offset;
}
};
static_assert(sizeof(VectorLocation) == sizeof(uint64_t),
"VectorLocation must be size of 8 bytes");
struct KeyInfo {
KeyInfo(void) {}
KeyInfo(uint32_t idx, const VectorLocation &loc)
: centroid_idx(idx), location(loc) {}
KeyInfo(VectorLocation loc) : location(loc) {}
uint32_t centroid_idx;
VectorLocation location;
};
// Segments ID
const std::string IVF_CENTROID_SEG_ID("ivf.centroid");
const std::string IVF_INVERTED_BODY_SEG_ID("ivf.inverted_body");
const std::string IVF_INVERTED_HEADER_SEG_ID("ivf.inverted_header");
const std::string IVF_INVERTED_META_SEG_ID("ivf.inverted_meta");
const std::string IVF_KEYS_SEG_ID("hc.keys");
const std::string IVF_OFFSETS_SEG_ID("ivf.offsets");
const std::string IVF_MAPPING_SEG_ID("ivf.mapping");
const std::string IVF_FEATURES_SEG_ID("ivf.features");
const std::string IVF_INT8_QUANTIZED_PARAMS_SEG_ID("ivf.int8_quantized_params");
const std::string IVF_INT4_QUANTIZED_PARAMS_SEG_ID("ivf.int4_quantized_params");
const std::string IVF_INVERTED_LIST_HEAD_SEG_ID("ivf.inverted_list_head");
const std::string IVF_STORAGE_SEGMENT_ID("ivf.S");
} // namespace core
} // namespace zvec
+110
View File
@@ -0,0 +1,110 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "framework/index_searcher.h"
#include "ivf_entity.h"
namespace zvec {
namespace core {
/*! IVF IndexProvider
*/
class IVFIndexProvider : public IndexProvider {
public:
IVFIndexProvider(const IndexMeta &meta, const IVFEntity::Pointer &entity,
const std::string &owner)
: meta_(meta), entity_(entity), owner_class_(owner) {}
IVFIndexProvider(const IVFIndexProvider &) = delete;
IVFIndexProvider &operator=(const IVFIndexProvider &) = delete;
public:
//! Create a new iterator
virtual Iterator::Pointer create_iterator(void) override {
return Iterator::Pointer(new (std::nothrow) Iterator(entity_));
}
//! Retrieve count of vectors
virtual size_t count(void) const override {
return entity_->vector_count();
}
//! Retrieve dimension of vector
virtual size_t dimension(void) const override {
return meta_.dimension();
}
//! Retrieve type of vector
virtual IndexMeta::DataType data_type(void) const override {
return meta_.data_type();
}
//! Retrieve vector size in bytes
virtual size_t element_size(void) const override {
return meta_.element_size();
}
//! Retrieve a vector using a primary key
virtual const void *get_vector(uint64_t key) const override {
return entity_->get_vector_by_key(key);
}
//! Retrieve the owner class
virtual const std::string &owner_class(void) const override {
return owner_class_;
}
private:
class Iterator : public IndexProvider::Iterator {
public:
Iterator(const IVFEntity::Pointer &entity) : entity_(entity) {}
//! Retrieve pointer of data
//! NOTICE: the vec feature will be changed after iterating to next, so
//! the caller need to keep a copy of it before iterator to next vector
virtual const void *data(void) const override {
return entity_->get_vector(index_);
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
return index_ < entity_->vector_count();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
return entity_->get_key(index_);
}
//! Next iterator
virtual void next(void) override {
++index_;
}
private:
//! Members
IVFEntity::Pointer entity_;
size_t index_{0};
};
private:
//! Members
const IndexMeta &meta_;
IVFEntity::Pointer entity_;
std::string owner_class_;
};
} // namespace core
} // namespace zvec
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
namespace zvec {
namespace core {
static const std::string SEPARATOR("/");
static const std::string CENTROID_SEPERATOR = "*";
// builder params
static const std::string PARAM_IVF_BUILDER_CENTROID_COUNT(
"proxima.ivf.builder.centroid_count");
static const std::string PARAM_IVF_BUILDER_CLUSTER_CLASS(
"proxima.ivf.builder.cluster_class");
static const std::string PARAM_IVF_BUILDER_THREAD_COUNT(
"proxima.ivf.builder.thread_count");
static const std::string PARAM_IVF_BUILDER_CLUSTER_AUTO_TUNING(
"proxima.ivf.builder.cluster_auto_tuning");
static const std::string PARAM_IVF_BUILDER_TRAIN_SAMPLE_COUNT(
"proxima.ivf.builder.train_sample_count");
static const std::string PARAM_IVF_BUILDER_TRAIN_SAMPLE_RATIO(
"proxima.ivf.builder.train_sample_ratio");
static const std::string PARAM_IVF_BUILDER_CONVERTER_PARAMS(
"proxima.ivf.builder.converter_params");
static const std::string PARAM_IVF_BUILDER_CONVERTER_CLASS(
"proxima.ivf.builder.converter_class");
static const std::string PARAM_IVF_BUILDER_STORE_ORIGINAL_FEATURES(
"proxima.ivf.builder.store_original_features");
static const std::string PARAM_IVF_BUILDER_QUANTIZER_CLASS(
"proxima.ivf.builder.quantizer_class");
static const std::string PARAM_IVF_BUILDER_QUANTIZE_BY_CENTROID(
"proxima.ivf.builder.quantize_by_centroid");
static const std::string PARAM_IVF_BUILDER_QUANTIZER_PARAMS(
"proxima.ivf.builder.quantizer_params");
static const std::string PARAM_IVF_BUILDER_CLUSTER_PARAMS_IN_LEVEL_PREFIX(
"proxima.ivf.builder.cluster_params_in_level_");
static const std::string PARAM_IVF_BUILDER_OPTIMIZER_CLASS(
"proxima.ivf.builder.optimizer_class");
static const std::string PARAM_IVF_BUILDER_OPTIMIZER_PARAMS(
"proxima.ivf.builder.optimizer_params");
static const std::string PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_CLASS(
"proxima.ivf.builder.optimizer_quantizer_class");
static const std::string PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_PARAMS(
"proxima.ivf.builder.optimizer_quantizer_params");
static const std::string PARAM_IVF_BUILDER_BLOCK_VECTOR_COUNT(
"proxima.ivf.builder.block_vector_count");
// searcher params
static const std::string PARAM_IVF_SEARCHER_SCAN_RATIO(
"proxima.ivf.searcher.scan_ratio");
static const std::string PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD(
"proxima.ivf.searcher.brute_force_threshold");
static const std::string PARAM_IVF_SEARCHER_OPTIMIZER(
"proxima.ivf.searcher.optimizer");
static const std::string PARAM_IVF_SEARCHER_OPTIMIZER_PARAMS(
"proxima.ivf.searcher.optimizer_params");
static const std::string PARAM_IVF_SEARCHER_CONVERTER_REFORMER(
"proxima.ivf.searcher.converter_reformer");
// Constants
static constexpr char const *kIPMetricName = "InnerProduct";
static constexpr char const *kMipsMetricName = "MipsSquaredEuclidean";
static constexpr char const *kL2MetricName = "SquaredEuclidean";
static constexpr char const *kMipsConverterName = "MipsConverter";
static constexpr char const *kMipsRevConverterName = "MipsReverseConverter";
static constexpr char const *kMipsReformerName = "MipsReformer";
static constexpr char const *kInt8QuantizerName = "Int8QuantizerConverter";
static constexpr char const *kInt4QuantizerName = "Int4QuantizerConverter";
static constexpr char const *kInt8ReformerName = "Int8QuantizerReformer";
static constexpr char const *kInt4ReformerName = "Int4QuantizerReformer";
static constexpr float kNormalizeScaleFactor = 16.0f;
} // namespace core
} // namespace zvec
+317
View File
@@ -0,0 +1,317 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ivf_searcher.h"
#include <ailego/utility/time_helper.h>
#include <framework/index_segment_storage.h>
#include "ivf_centroid_index.h"
#include "ivf_index_provider.h"
#include "ivf_params.h"
namespace zvec {
namespace core {
int IVFSearcher::init(const ailego::Params &parameters) {
params_ = parameters;
params_.get(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD, &bruteforce_threshold_);
searcher_state_ = STATE_INITED;
return 0;
}
int IVFSearcher::cleanup(void) {
this->unload();
params_.clear();
bruteforce_threshold_ = kDefaultBfThreshold;
searcher_state_ = STATE_INIT;
return 0;
}
int IVFSearcher::load(IndexStorage::Pointer container,
IndexMetric::Pointer /*metric*/) {
if (!container) {
LOG_ERROR("Invalid container");
return IndexError_InvalidArgument;
}
if (searcher_state_ != STATE_INITED) {
LOG_ERROR("Initalize the searcher first before load index");
return IndexError_Runtime;
}
ailego::ElapsedTime timer;
int ret = IndexHelper::DeserializeFromStorage(container.get(), &meta_);
if (ret != 0) {
LOG_ERROR("Failed to deserialize meta from container");
return ret;
}
//! Load centroid index
centroid_index_ = std::make_shared<IVFCentroidIndex>();
if (!centroid_index_) {
return IndexError_NoMemory;
}
auto seg = container->get(IVF_CENTROID_SEG_ID, 0);
if (!seg) {
LOG_ERROR("Failed to get segment %s", IVF_CENTROID_SEG_ID.c_str());
return IndexError_InvalidFormat;
}
IndexStorage::Pointer seg_container =
std::make_shared<IndexSegmentStorage>(seg);
if (!seg_container) {
return IndexError_NoMemory;
}
ret = seg_container->open(std::string(), false);
if (ret != 0) {
LOG_ERROR("IndexSegmentStorage load failed for %s", IndexError::What(ret));
return ret;
}
ret = centroid_index_->load(seg_container, params_);
if (ret != 0) {
LOG_ERROR("Failed to load index for %s", IndexError::What(ret));
return ret;
}
auto reformer = centroid_index_->reformer();
params_.set(PARAM_IVF_SEARCHER_CONVERTER_REFORMER, reformer);
//! load iverted index
entity_ = std::make_shared<IVFEntity>();
if (!entity_) {
return IndexError_NoMemory;
}
ret = entity_->load(container);
ivf_check_error_code(ret);
magic_ = IndexContext::GenerateMagic();
stats_.set_loaded_count(entity_->vector_count());
stats_.set_loaded_costtime(timer.milli_seconds());
searcher_state_ = STATE_LOADED;
return 0;
}
int IVFSearcher::unload(void) {
magic_ = 0;
centroid_index_.reset();
entity_.reset();
stats_.set_loaded_count(0UL);
stats_.set_loaded_costtime(0UL);
stats_.clear_attributes();
searcher_state_ = STATE_INITED;
return 0;
}
int IVFSearcher::search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const {
return search_bf_impl(query, qmeta, 1, context);
}
int IVFSearcher::search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const {
if (!query || qmeta.element_size() != meta_.element_size()) {
LOG_ERROR("Null query or invalid qmeta");
return IndexError_InvalidArgument;
}
IVFSearcherContext *ctx = dynamic_cast<IVFSearcherContext *>(context.get());
if (!ctx || ctx->topk() == 0) {
LOG_ERROR("Invalid context or topk not set yet");
return IndexError_InvalidArgument;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher
int ret = this->update_context(ctx);
ivf_check_error_code(ret);
}
ctx->reset_results(count);
auto &entity = ctx->entity();
auto &filter = ctx->filter();
//! Transform the querys for querying in inverted vector index later
IndexQueryMeta iv_qmeta;
int ret = entity->transform(query, qmeta, count, &query, &iv_qmeta);
ivf_check_with_msg(ret, "Failed to transform querys");
// TODO: do batch search in matrix
for (size_t q = 0; q < count; ++q) {
auto &context_stats = ctx->mutable_stats(q);
auto &heap = ctx->mutable_result_heap();
heap.clear();
if (!filter.is_valid()) {
ret = entity->search(query, &heap, &context_stats);
} else {
ret = entity->search(query, filter, &heap, &context_stats);
}
ivf_check_with_msg(ret, "Failed to search in entity for %s",
IndexError::What(ret));
heap.sort(); // sort the results
if (!filter.is_valid()) {
// mapping the local id to key if query without filter
ret = entity->retrieve_keys(&heap);
ivf_check_error_code(ret);
}
entity->normalize(q, &heap);
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + iv_qmeta.element_size();
}
return 0;
}
int IVFSearcher::search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const {
return this->search_impl(query, qmeta, 1, context);
}
int IVFSearcher::search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const {
if (entity_->vector_count() <= bruteforce_threshold_) {
return this->search_bf_impl(query, qmeta, count, context);
}
if (!query || qmeta.element_size() != meta_.element_size()) {
LOG_ERROR("Null query or invalid qmeta");
return IndexError_InvalidArgument;
}
IVFSearcherContext *ctx = dynamic_cast<IVFSearcherContext *>(context.get());
if (!ctx || ctx->topk() == 0) {
LOG_ERROR("Invalid context or topk not set yet");
return IndexError_InvalidArgument;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher
int ret = update_context(ctx);
ivf_check_error_code(ret);
}
ctx->reset_results(count);
auto &entity = ctx->entity();
auto &filter = ctx->filter();
auto &centroid_index_ctx = ctx->centroid_searcher_ctx();
int ret = centroid_index_->search(query, qmeta, count, centroid_index_ctx);
ivf_check_error_code(ret);
//! Transform the querys for querying in inverted vector index later
IndexQueryMeta iv_qmeta;
ret = entity->transform(query, qmeta, count, &query, &iv_qmeta);
ivf_check_with_msg(ret, "Failed to transform querys");
for (size_t q = 0; q < count; ++q) {
auto &centroids = centroid_index_ctx->result(q);
auto &context_stats = ctx->mutable_stats(q);
auto &heap = ctx->mutable_result_heap();
heap.clear();
uint32_t total_scan_count = 0;
for (size_t i = 0;
i < centroids.size() && total_scan_count < ctx->max_scan_count();
++i) {
auto cid = centroids[i].key();
uint32_t scan_count = 0;
if (!filter.is_valid()) {
ret = entity->search(cid, query, &scan_count, &heap, &context_stats);
} else {
ret = entity->search(cid, query, filter, &scan_count, &heap,
&context_stats);
}
ivf_check_with_msg(ret, "Failed to search in entity for %s",
IndexError::What(ret));
total_scan_count += scan_count;
}
heap.sort(); // sort the results
if (!filter.is_valid()) {
// mapping the local id to key if query without filter
ret = entity->retrieve_keys(&heap);
ivf_check_error_code(ret);
}
entity->normalize(q, &heap);
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + iv_qmeta.element_size();
}
return 0;
}
const IndexSearcher::Stats &IVFSearcher::stats(void) const {
return stats_;
}
IndexSearcher::Context::Pointer IVFSearcher::create_context() const {
if (searcher_state_ != STATE_LOADED) {
LOG_ERROR("Load the index first before create context");
return nullptr;
}
auto entity = entity_->clone();
if (!entity) {
LOG_ERROR("Failed to clone IVFEntity");
return nullptr;
}
auto centroid_index_ctx = centroid_index_->create_context();
if (!centroid_index_ctx) {
LOG_ERROR("Failed to create centroid index context");
return nullptr;
}
auto context =
new (std::nothrow) IVFSearcherContext(entity, centroid_index_ctx);
if (!context) {
LOG_ERROR("Failed to alloc IVFSearcherContext");
return nullptr;
}
int ret = context->init(params_);
if (ret != 0) {
delete context;
return nullptr;
}
context->set_magic(magic_);
return Context::Pointer(context);
}
IndexProvider::Pointer IVFSearcher::create_provider(void) const {
if (searcher_state_ != STATE_LOADED) {
LOG_ERROR("Load the index first before create provider");
return nullptr;
}
auto entity = entity_->clone();
if (!entity) {
LOG_ERROR("Failed to clone IVFEntity");
return Provider::Pointer();
}
auto *provider = new (std::nothrow)
IVFIndexProvider(entity->has_orignal_feature() ? meta_ : entity->meta(),
entity, "IVFSearcher");
if (!provider) {
LOG_ERROR("Failed to alloc IVFIndexProvider");
return Provider::Pointer();
}
return Provider::Pointer(provider);
}
INDEX_FACTORY_REGISTER_SEARCHER(IVFSearcher);
} // namespace core
} // namespace zvec
+114
View File
@@ -0,0 +1,114 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "framework/index_searcher.h"
#include "ivf_centroid_index.h"
#include "ivf_entity.h"
#include "ivf_searcher_context.h"
namespace zvec {
namespace core {
/*! IVF Searcher
*/
class IVFSearcher : public IndexSearcher {
public:
//! Initialize Searcher
virtual int init(const ailego::Params &parameters) override;
//! Cleanup Searcher
virtual int cleanup(void) override;
//! Load index from container
virtual int load(IndexStorage::Pointer container,
IndexMetric::Pointer metric) override;
//! Unload index
virtual int unload(void) override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
//! Retrieve statistics
virtual const Stats &stats(void) const override;
//! Create a searcher context
virtual Context::Pointer create_context(void) const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
return meta_;
}
//! Retrieve params of index
virtual const ailego::Params &params(void) const override {
return params_;
}
protected:
int update_context(IVFSearcherContext *ctx) const {
auto entity = entity_->clone();
if (!entity) {
LOG_ERROR("Failed to clone QcEntity");
return IndexError_Runtime;
}
//! The centroid index searcher may be different, so need to create one
auto centroid_ctx = centroid_index_->create_context();
if (!centroid_ctx) {
LOG_ERROR("Failed to create centroid index searcher context");
return IndexError_Runtime;
}
return ctx->update_context(entity, centroid_ctx, params_, magic_);
}
private:
//! Constants
static constexpr uint32_t kDefaultBfThreshold = 1000u;
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
//! Members
IndexMeta meta_{};
ailego::Params params_{};
IVFCentroidIndex::Pointer centroid_index_{};
IVFEntity::Pointer entity_{};
uint32_t bruteforce_threshold_{kDefaultBfThreshold};
uint32_t magic_{0};
Stats stats_{};
State searcher_state_{STATE_INIT};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,223 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/container/heap.h>
#include "ivf_entity.h"
#include "ivf_utility.h"
namespace zvec {
namespace core {
/*! IVF Searcher Context
*/
class IVFSearcherContext : public IndexSearcher::Context {
public:
IVFSearcherContext(const IVFEntity::Pointer &ivf_entity,
IndexSearcher::Context::Pointer &centroid_ctx)
: entity_(ivf_entity), centroid_searcher_ctx_(std::move(centroid_ctx)) {}
public:
//! Set topk of search result
virtual void set_topk(uint32_t k) override {
topk_ = k;
result_heap_.limit(topk_);
result_heap_.set_threshold(this->threshold());
}
//! Retrieve search result
virtual const IndexDocumentList &result(void) const override {
return results_[0];
}
//! Retrieve search result with index
virtual const IndexDocumentList &result(size_t idx) const override {
ailego_assert_with(results_.size() > idx, "invalid index");
return results_[idx];
}
//! Retrieve mutable result with index
virtual IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
inline IndexDocumentHeap *result_heap() {
return &result_heap_;
}
//! Update the parameters of context
virtual int update(const ailego::Params &params) override {
params.get(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
params.get(PARAM_IVF_SEARCHER_SCAN_RATIO, &scan_ratio_);
if (scan_ratio_ <= 0.0) {
LOG_ERROR("Invalid params %s=%f", PARAM_IVF_SEARCHER_SCAN_RATIO.c_str(),
scan_ratio_);
return IndexError_InvalidArgument;
}
size_t topk_val =
std::max(static_cast<uint32_t>(
std::round(entity_->inverted_list_count() * scan_ratio_)),
1u);
centroid_searcher_ctx_->set_topk(topk_val);
max_scan_count_ =
static_cast<uint32_t>(std::ceil(entity_->vector_count() * scan_ratio_));
max_scan_count_ = std::max(bruteforce_threshold_, max_scan_count_);
return 0;
}
//! Retrieve magic number
virtual uint32_t magic(void) const override {
return magic_;
}
public:
//! Initialize the context
int init(const ailego::Params &params) {
return this->update(params);
}
//! Update the magic number
void set_magic(uint32_t mag) {
magic_ = mag;
}
//! Get Topk Value
uint32_t topk() const override {
return topk_;
}
//! Retrieve scan ratio
float scan_ratio(void) const {
return scan_ratio_;
}
//! Retrieve max scan count
uint32_t max_scan_count(void) const {
return max_scan_count_;
}
uint32_t bruteforce_threshold() const {
return bruteforce_threshold_;
}
//! Retrieve magic number
const IVFEntity::Pointer &entity() const {
return entity_;
}
//! Retrieve Mutable Query Result By Query Index
IndexDocumentHeap &mutable_result_heap() {
return result_heap_;
}
void set_fetch_vector(bool v) override {
fetch_vector_ = v;
}
bool fetch_vector(void) const override {
return fetch_vector_;
}
//! Reset all the query results
void reset_results(size_t qnum) {
results_.resize(qnum);
stats_vec_.resize(qnum);
for (size_t i = 0; i < qnum; ++i) {
results_[i].clear();
stats_vec_[i].clear();
}
result_heap_.clear();
result_heap_.limit(topk_);
result_heap_.set_threshold(this->threshold());
}
//! Update context, the context may be shared by different searcher
int update_context(IVFEntity::Pointer &new_entity,
IndexSearcher::Context::Pointer &centroid_ctx,
const ailego::Params &params, uint32_t magic_num) {
entity_ = new_entity;
centroid_searcher_ctx_ = std::move(centroid_ctx);
int ret = this->update(params);
ivf_check_error_code(ret);
magic_ = magic_num;
return 0;
}
//! Retrieve the centroid index context
IndexSearcher::Context::Pointer &centroid_searcher_ctx(void) {
return centroid_searcher_ctx_;
}
const Stats &stats(size_t idx = 0) const {
ailego_assert_with(stats_vec_.size() > idx, "invalid index");
return stats_vec_[idx];
}
Stats &mutable_stats(size_t idx = 0) {
ailego_assert_with(stats_vec_.size() > idx, "invalid index");
return stats_vec_[idx];
}
void topk_to_result(uint32_t idx) {
if (ailego_unlikely(result_heap_.size() == 0)) {
return;
}
ailego_assert_with(idx < results_.size(), "invalid idx");
int size = std::min(topk_, static_cast<uint32_t>(result_heap_.size()));
result_heap_.sort();
results_[idx].clear();
for (int i = 0; i < size; ++i) {
auto score = result_heap_[i].score();
if (score > this->threshold()) {
break;
}
key_t key = result_heap_[i].key();
if (fetch_vector_) {
IndexStorage::MemoryBlock block;
entity_->get_vector_by_key(key, block);
results_[idx].emplace_back(key, score, key, block);
} else {
results_[idx].emplace_back(key, score);
}
}
}
private:
//! Constants
static constexpr float kDefaultScanRatio = 0.1f;
static constexpr uint32_t kDefaultBfThreshold = 1000u;
//! Members
IVFEntity::Pointer entity_{};
IndexSearcher::Context::Pointer centroid_searcher_ctx_{};
IndexDocumentHeap result_heap_;
std::vector<IndexDocumentList> results_{};
std::vector<Stats> stats_vec_{};
bool fetch_vector_{false};
uint32_t topk_{0};
uint32_t magic_{0};
float scan_ratio_{kDefaultScanRatio};
uint32_t max_scan_count_{0};
uint32_t bruteforce_threshold_{kDefaultBfThreshold};
};
} // namespace core
} // namespace zvec
+317
View File
@@ -0,0 +1,317 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ivf_streamer.h"
#include <ailego/utility/time_helper.h>
#include <framework/index_segment_storage.h>
#include "ivf_centroid_index.h"
#include "ivf_index_provider.h"
#include "ivf_params.h"
namespace zvec {
namespace core {
int IVFStreamer::init(const IndexMeta &meta, const ailego::Params &parameters) {
meta_ = meta;
params_ = parameters;
params_.get(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD, &bruteforce_threshold_);
searcher_state_ = STATE_INITED;
return 0;
}
int IVFStreamer::cleanup(void) {
this->unload();
params_.clear();
bruteforce_threshold_ = kDefaultBfThreshold;
searcher_state_ = STATE_INIT;
return 0;
}
int IVFStreamer::open(IndexStorage::Pointer storage) {
if (!storage) {
LOG_ERROR("Invalid storage");
return IndexError_InvalidArgument;
}
if (searcher_state_ != STATE_INITED) {
LOG_ERROR("Initalize the searcher first before load index");
return IndexError_Runtime;
}
ailego::ElapsedTime timer;
int ret = IndexHelper::DeserializeFromStorage(storage.get(), &meta_);
if (ret != 0) {
LOG_ERROR("Failed to deserialize meta from storage");
return ret;
}
//! Load centroid index
centroid_index_ = std::make_shared<IVFCentroidIndex>();
if (!centroid_index_) {
return IndexError_NoMemory;
}
auto seg = storage->get(IVF_CENTROID_SEG_ID, 0);
if (!seg) {
LOG_ERROR("Failed to get segment %s", IVF_CENTROID_SEG_ID.c_str());
return IndexError_InvalidFormat;
}
IndexStorage::Pointer seg_container =
std::make_shared<IndexSegmentStorage>(seg);
if (!seg_container) {
return IndexError_NoMemory;
}
ret = seg_container->open(std::string(), false);
if (ret != 0) {
LOG_ERROR("IndexSegmentStorage load failed for %s", IndexError::What(ret));
return ret;
}
ret = centroid_index_->load(seg_container, params_);
if (ret != 0) {
LOG_ERROR("Failed to load index for %s", IndexError::What(ret));
return ret;
}
auto reformer = centroid_index_->reformer();
params_.set(PARAM_IVF_SEARCHER_CONVERTER_REFORMER, reformer);
//! load iverted index
entity_ = std::make_shared<IVFEntity>();
if (!entity_) {
return IndexError_NoMemory;
}
ret = entity_->load(storage);
ivf_check_error_code(ret);
magic_ = IndexContext::GenerateMagic();
stats_.set_loaded_count(entity_->vector_count());
stats_.set_loaded_costtime(timer.milli_seconds());
searcher_state_ = STATE_LOADED;
return 0;
}
int IVFStreamer::unload(void) {
magic_ = 0;
centroid_index_.reset();
entity_.reset();
stats_.set_loaded_count(0UL);
stats_.set_loaded_costtime(0UL);
stats_.clear_attributes();
searcher_state_ = STATE_INITED;
return 0;
}
int IVFStreamer::search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const {
return search_bf_impl(query, qmeta, 1, context);
}
int IVFStreamer::search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const {
if (!query || qmeta.element_size() != meta_.element_size()) {
LOG_ERROR("Null query or invalid qmeta");
return IndexError_InvalidArgument;
}
IVFSearcherContext *ctx = dynamic_cast<IVFSearcherContext *>(context.get());
if (!ctx || ctx->topk() == 0) {
LOG_ERROR("Invalid context or topk not set yet");
return IndexError_InvalidArgument;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher
int ret = this->update_context(ctx);
ivf_check_error_code(ret);
}
ctx->reset_results(count);
auto &entity = ctx->entity();
auto &filter = ctx->filter();
//! Transform the querys for querying in inverted vector index later
IndexQueryMeta iv_qmeta;
int ret = entity->transform(query, qmeta, count, &query, &iv_qmeta);
ivf_check_with_msg(ret, "Failed to transform querys");
// TODO: do batch search in matrix
for (size_t q = 0; q < count; ++q) {
auto &context_stats = ctx->mutable_stats(q);
auto &heap = ctx->mutable_result_heap();
heap.clear();
if (!filter.is_valid()) {
ret = entity->search(query, &heap, &context_stats);
} else {
ret = entity->search(query, filter, &heap, &context_stats);
}
ivf_check_with_msg(ret, "Failed to search in entity for %s",
IndexError::What(ret));
heap.sort(); // sort the results
if (!filter.is_valid()) {
// mapping the local id to key if query without filter
ret = entity->retrieve_keys(&heap);
ivf_check_error_code(ret);
}
entity->normalize(q, &heap);
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + iv_qmeta.element_size();
}
return 0;
}
int IVFStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const {
return this->search_impl(query, qmeta, 1, context);
}
int IVFStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const {
if (entity_->vector_count() <= bruteforce_threshold_) {
return this->search_bf_impl(query, qmeta, count, context);
}
if (!query || qmeta.element_size() != meta_.element_size()) {
LOG_ERROR("Null query or invalid qmeta");
return IndexError_InvalidArgument;
}
IVFSearcherContext *ctx = dynamic_cast<IVFSearcherContext *>(context.get());
if (!ctx || ctx->topk() == 0) {
LOG_ERROR("Invalid context or topk not set yet");
return IndexError_InvalidArgument;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher
int ret = update_context(ctx);
ivf_check_error_code(ret);
}
ctx->reset_results(count);
auto &entity = ctx->entity();
auto &filter = ctx->filter();
auto &centroid_index_ctx = ctx->centroid_searcher_ctx();
int ret = centroid_index_->search(query, qmeta, count, centroid_index_ctx);
ivf_check_error_code(ret);
//! Transform the querys for querying in inverted vector index later
IndexQueryMeta iv_qmeta;
ret = entity->transform(query, qmeta, count, &query, &iv_qmeta);
ivf_check_with_msg(ret, "Failed to transform querys");
for (size_t q = 0; q < count; ++q) {
auto &centroids = centroid_index_ctx->result(q);
auto &context_stats = ctx->mutable_stats(q);
auto &heap = ctx->mutable_result_heap();
heap.clear();
uint32_t total_scan_count = 0;
for (size_t i = 0;
i < centroids.size() && total_scan_count < ctx->max_scan_count();
++i) {
auto cid = centroids[i].key();
uint32_t scan_count = 0;
if (!filter.is_valid()) {
ret = entity->search(cid, query, &scan_count, &heap, &context_stats);
} else {
ret = entity->search(cid, query, filter, &scan_count, &heap,
&context_stats);
}
ivf_check_with_msg(ret, "Failed to search in entity for %s",
IndexError::What(ret));
total_scan_count += scan_count;
}
heap.sort(); // sort the results
if (!filter.is_valid()) {
// mapping the local id to key if query without filter
ret = entity->retrieve_keys(&heap);
ivf_check_error_code(ret);
}
entity->normalize(q, &heap);
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + iv_qmeta.element_size();
}
return 0;
}
const IndexSearcher::Stats &IVFStreamer::stats(void) const {
return stats_;
}
IndexSearcher::Context::Pointer IVFStreamer::create_context() const {
if (searcher_state_ != STATE_LOADED) {
LOG_ERROR("Load the index first before create context");
return nullptr;
}
auto entity = entity_->clone();
if (!entity) {
LOG_ERROR("Failed to clone IVFEntity");
return nullptr;
}
auto centroid_index_ctx = centroid_index_->create_context();
if (!centroid_index_ctx) {
LOG_ERROR("Failed to create centroid index context");
return nullptr;
}
auto context =
new (std::nothrow) IVFSearcherContext(entity, centroid_index_ctx);
if (!context) {
LOG_ERROR("Failed to alloc IVFSearcherContext");
return nullptr;
}
int ret = context->init(params_);
if (ret != 0) {
delete context;
return nullptr;
}
context->set_magic(magic_);
return Context::Pointer(context);
}
IndexProvider::Pointer IVFStreamer::create_provider(void) const {
if (searcher_state_ != STATE_LOADED) {
LOG_ERROR("Load the index first before create provider");
return nullptr;
}
auto entity = entity_->clone();
if (!entity) {
LOG_ERROR("Failed to clone IVFEntity");
return Provider::Pointer();
}
auto *provider = new (std::nothrow)
IVFIndexProvider(entity->has_orignal_feature() ? meta_ : entity->meta(),
entity, "IVFStreamer");
if (!provider) {
LOG_ERROR("Failed to alloc IVFIndexProvider");
return Provider::Pointer();
}
return Provider::Pointer(provider);
}
INDEX_FACTORY_REGISTER_STREAMER(IVFStreamer);
} // namespace core
} // namespace zvec
+125
View File
@@ -0,0 +1,125 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __IVF_STREAMER_H__
#define __IVF_STREAMER_H__
#include "framework/index_streamer.h"
#include "ivf_centroid_index.h"
#include "ivf_entity.h"
#include "ivf_searcher_context.h"
namespace zvec {
namespace core {
/*! IVF Searcher
*/
class IVFStreamer : public IndexStreamer {
public:
//! Initialize Searcher
virtual int init(const IndexMeta & /*meta*/,
const ailego::Params & /*params*/) override;
//! Cleanup Searcher
virtual int cleanup(void) override;
//! Load index from container
virtual int open(IndexStorage::Pointer storage) override;
virtual int flush(uint64_t /*check_point*/) override {
return 0;
}
virtual int close(void) override {
return this->unload();
}
//! Unload index
virtual int unload(void) override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
//! Retrieve statistics
virtual const Stats &stats(void) const override;
//! Create a searcher context
virtual Context::Pointer create_context(void) const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
return meta_;
}
virtual int get_vector_by_id(
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector_by_key(id, block);
}
protected:
int update_context(IVFSearcherContext *ctx) const {
auto entity = entity_->clone();
if (!entity) {
LOG_ERROR("Failed to clone QcEntity");
return IndexError_Runtime;
}
//! The centroid index searcher may be different, so need to create one
auto centroid_ctx = centroid_index_->create_context();
if (!centroid_ctx) {
LOG_ERROR("Failed to create centroid index searcher context");
return IndexError_Runtime;
}
return ctx->update_context(entity, centroid_ctx, params_, magic_);
}
private:
//! Constants
static constexpr uint32_t kDefaultBfThreshold = 1000u;
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
//! Members
IndexMeta meta_{};
ailego::Params params_{};
IndexBuilder::Pointer builder_;
IVFCentroidIndex::Pointer centroid_index_{};
IVFEntity::Pointer entity_{};
uint32_t bruteforce_threshold_{kDefaultBfThreshold};
uint32_t magic_{0};
Stats stats_{};
State searcher_state_{STATE_INIT};
};
} // namespace core
} // namespace zvec
#endif //__IVF_STREAMER_H__
+177
View File
@@ -0,0 +1,177 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <algorithm>
#include <memory>
#include <mutex>
#include <numeric>
#include <vector>
#include <ailego/utility/matrix_helper.h>
#include <ailego/utility/time_helper.h>
namespace zvec {
namespace core {
#ifndef ivf_check_error_code
#define ivf_check_error_code(code) \
if (ailego_unlikely((code) != 0)) return code
#endif
#ifndef ivf_assert
#define ivf_assert(cond, code) \
if (ailego_unlikely(!(cond))) return code
#endif
#ifndef ivf_check_with_msg
#define ivf_check_with_msg(code, fmt, args...) \
do { \
if (ailego_unlikely((code) != 0)) { \
LOG_ERROR(fmt, ##args); \
return code; \
} \
} while (0)
#endif
#ifndef ivf_assert_with_msg
#define ivf_assert_with_msg(cond, err, fmt, args...) \
do { \
if (ailego_unlikely(!(cond))) { \
LOG_ERROR(fmt, ##args); \
return err; \
} \
} while (0)
#endif
/*! Quantized Clustering Utility
*/
class IVFUtility {
public:
//! Generator a random path with specificed prefix
static inline std::string GenerateRandomPath(const std::string &prefix) {
uint64_t timestamp = ailego::Monotime::MicroSeconds();
return prefix + std::to_string(timestamp);
}
//! Compute the default scan ratio for total vectors
static inline float ComputeScanRatio(size_t vector_count) {
// the fitting function for the follow points: 1000000(0.02)
// 10000000(0.01) 50000000(0.005) 100000000(0.001)
float scan_ratio = -0.004 * std::log(vector_count) + 0.0751;
scan_ratio = std::max(scan_ratio, 0.0001f);
return scan_ratio;
}
//! Transpose the vectors in row major order to column major order
static inline void Transpose(size_t align_size, const void *src, size_t m,
size_t dim, void *dst);
//! Transpose the vectors in column major order to row major order
static inline void ReverseTranspose(size_t align_size, const void *src,
size_t m, size_t dim, void *dst);
//! Aligned size of a block vectors buffer
static inline size_t AlignedSize(size_t fnum, size_t element_size);
//! Aligned size of one vector buffer
static inline size_t AlignedSize(size_t element_size);
//! Sort arr with size in ascending order, and keep the index postion
//! n2o keep the mapping: new position => origin postion
//! For example, the input arr = [5, 3, 9, 6, 7], size = 5, after sort
// arr = [3, 5, 6, 7, 9]
// n2o = [1, 0, 3, 4, 2]
//! To save memory, no extra memory is allocated
template <typename T, typename I>
static void Sort(T *arr, std::vector<I> *n2o, size_t size) {
std::vector<I> o2n;
o2n.resize(size);
n2o->resize(size);
std::iota(n2o->begin(), n2o->end(), 0U);
std::sort(n2o->begin(), n2o->end(),
[&](I i, I j) { return arr[i] < arr[j]; });
for (I i = 0U; i < size; ++i) {
o2n[(*n2o)[i]] = i;
}
//! reorder arr in place, according to given n2o index
for (I i = 0; i < size; ++i) {
if (i != (*n2o)[i]) {
T tmp = arr[i];
I j = i, k;
while (i != (k = (*n2o)[j])) {
arr[j] = arr[k];
(*n2o)[j] = j;
j = k;
}
arr[j] = tmp;
(*n2o)[j] = j;
}
}
for (I i = 0U; i < size; ++i) {
(*n2o)[o2n[i]] = i;
}
}
//! Transpose one vector in block
template <typename T>
static inline void TransposeOne(const void *src, size_t M, size_t N,
void *dst) {
for (size_t i = 0; i < N; ++i) {
reinterpret_cast<T *>(dst)[i] = reinterpret_cast<const T *>(src)[i * M];
}
}
};
void IVFUtility::Transpose(size_t align_size, const void *src, size_t m,
size_t dim, void *dst) {
switch (align_size) {
case 2:
ailego::MatrixHelper::Transpose<uint16_t>(src, m, dim, dst);
break;
case 4:
ailego::MatrixHelper::Transpose<uint32_t>(src, m, dim, dst);
break;
case 8:
ailego::MatrixHelper::Transpose<uint64_t>(src, m, dim, dst);
break;
}
}
void IVFUtility::ReverseTranspose(size_t align_size, const void *src, size_t m,
size_t dim, void *dst) {
switch (align_size) {
case 2:
ailego::MatrixHelper::ReverseTranspose<uint16_t>(src, m, dim, dst);
break;
case 4:
ailego::MatrixHelper::ReverseTranspose<uint32_t>(src, m, dim, dst);
break;
case 8:
ailego::MatrixHelper::ReverseTranspose<uint64_t>(src, m, dim, dst);
break;
}
}
size_t IVFUtility::AlignedSize(size_t fnum, size_t element_size) {
return ailego_align(fnum * element_size, 32);
}
size_t IVFUtility::AlignedSize(size_t element_size) {
return ailego_align(element_size, 32);
}
} // namespace core
} // namespace zvec