Initial commit
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
|
||||
cc_library(
|
||||
NAME core_knn_ivf STATIC SHARED STRICT ALWAYS_LINK
|
||||
SRCS *.cc
|
||||
LIBS zvec_ailego core_framework core_knn_cluster
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
@@ -0,0 +1,829 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "ivf_builder.h"
|
||||
#include <ailego/pattern/defer.h>
|
||||
#include <ailego/utility/string_helper.h>
|
||||
#include "cluster/cluster_params.h"
|
||||
#include "ivf_dumper.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! IndexHolder support filtered by vector labels
|
||||
*/
|
||||
class LabelFilteredIndexHolder : public IndexHolder {
|
||||
public:
|
||||
/*! Index Holder Iterator
|
||||
*/
|
||||
class Iterator : public IndexHolder::Iterator {
|
||||
public:
|
||||
//! Index Holder Iterator Pointer
|
||||
typedef std::unique_ptr<Iterator> Pointer;
|
||||
|
||||
//! Constructor
|
||||
Iterator(const IVFBuilder::RandomAccessIndexHolder::Pointer &holder,
|
||||
const std::vector<uint32_t> *elems)
|
||||
: holder_(holder), elems_(elems) {}
|
||||
|
||||
//! Destructor
|
||||
virtual ~Iterator(void) {}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
virtual const void *data(void) const override {
|
||||
return holder_->element((*elems_)[index_]);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
return index_ < elems_->size();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
return (*elems_)[index_];
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
++index_;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
const IVFBuilder::RandomAccessIndexHolder::Pointer holder_{nullptr};
|
||||
const std::vector<uint32_t> *elems_{nullptr};
|
||||
size_t index_{0};
|
||||
};
|
||||
|
||||
//! Constructor
|
||||
LabelFilteredIndexHolder(
|
||||
const IVFBuilder::RandomAccessIndexHolder::Pointer &holder,
|
||||
const std::vector<uint32_t> &items)
|
||||
: holder_(holder), elems_(&items) {}
|
||||
|
||||
//! Retrieve count of elements in holder (-1 indicates unknown)
|
||||
virtual size_t count(void) const override {
|
||||
return elems_->size();
|
||||
}
|
||||
|
||||
//! Retrieve dimension
|
||||
virtual size_t dimension(void) const override {
|
||||
return holder_->dimension();
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
virtual IndexMeta::DataType data_type(void) const override {
|
||||
return holder_->data_type();
|
||||
}
|
||||
|
||||
//! Retrieve element size in bytes
|
||||
virtual size_t element_size(void) const override {
|
||||
return holder_->element_size();
|
||||
}
|
||||
|
||||
//! Retrieve if it can multi-pass
|
||||
virtual bool multipass(void) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
|
||||
return IndexHolder::Iterator::Pointer(
|
||||
new LabelFilteredIndexHolder::Iterator(holder_, elems_));
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
const IVFBuilder::RandomAccessIndexHolder::Pointer holder_{};
|
||||
const std::vector<uint32_t> *elems_{};
|
||||
};
|
||||
|
||||
IVFBuilder::IVFBuilder() {}
|
||||
|
||||
IVFBuilder::~IVFBuilder() {
|
||||
this->cleanup();
|
||||
}
|
||||
|
||||
int IVFBuilder::init(const IndexMeta &meta, const ailego::Params ¶ms) {
|
||||
LOG_INFO("Begin IVFBuilder::init!");
|
||||
|
||||
if (state_ != INIT) {
|
||||
LOG_ERROR("IVFBuilder state wrong. state=%d", state_);
|
||||
return IndexError_Logic;
|
||||
}
|
||||
|
||||
meta_ = meta;
|
||||
converted_meta_ = meta;
|
||||
quantized_meta_ = meta;
|
||||
// Clear the converter/reformer params for external transforms
|
||||
converted_meta_.set_reformer(std::string(), 0, ailego::Params());
|
||||
converted_meta_.set_converter(std::string(), 0, ailego::Params());
|
||||
quantized_meta_.set_reformer(std::string(), 0, ailego::Params());
|
||||
quantized_meta_.set_converter(std::string(), 0, ailego::Params());
|
||||
params_ = params;
|
||||
|
||||
if (!IndexFactory::HasMetric(meta_.metric_name())) {
|
||||
LOG_ERROR("Metric %s not exist", meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
int ret = parse_centroids_num(params);
|
||||
ivf_check_with_msg(ret, "Failed to parse centroids, ret=%d", ret);
|
||||
|
||||
ret = parse_clustering_params(params);
|
||||
ivf_check_with_msg(ret, "Failed to parse clustering params, ret=%d", ret);
|
||||
|
||||
ret = parse_general_params(params);
|
||||
ivf_check_with_msg(ret, "Failed to parse general params, ret=%d", ret);
|
||||
|
||||
LOG_INFO("End IVFBuilder::init!");
|
||||
|
||||
LOG_DEBUG(
|
||||
"Converter=%s Quantizer=%s Optimizer=%s "
|
||||
"OptimizerQuantizer=%s QuantizeByCentroid=%u StoreFeatures=%u "
|
||||
"ClusterClass=%s TrainSamplesCount=%u TrainSampleRatio=%f "
|
||||
"BlockVectorCount=%u",
|
||||
params.get_as_string(PARAM_IVF_BUILDER_CONVERTER_CLASS).c_str(),
|
||||
params.get_as_string(PARAM_IVF_BUILDER_QUANTIZER_CLASS).c_str(),
|
||||
params.get_as_string(PARAM_IVF_BUILDER_OPTIMIZER_CLASS).c_str(),
|
||||
params.get_as_string(PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_CLASS).c_str(),
|
||||
params.get_as_bool(PARAM_IVF_BUILDER_QUANTIZE_BY_CENTROID),
|
||||
params.get_as_bool(PARAM_IVF_BUILDER_STORE_ORIGINAL_FEATURES),
|
||||
params.get_as_string(PARAM_IVF_BUILDER_CLUSTER_CLASS).c_str(),
|
||||
params.get_as_uint32(PARAM_IVF_BUILDER_TRAIN_SAMPLE_COUNT),
|
||||
params.get_as_float(PARAM_IVF_BUILDER_TRAIN_SAMPLE_RATIO),
|
||||
block_vector_count_);
|
||||
|
||||
state_ = INITED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::cleanup(void) {
|
||||
LOG_INFO("Begin IVFBuilder::cleanup");
|
||||
|
||||
state_ = INIT;
|
||||
stats_.clear_attributes();
|
||||
stats_.set_built_costtime(0u);
|
||||
stats_.set_built_count(0u);
|
||||
stats_.set_discarded_count(0u);
|
||||
stats_.set_dumped_costtime(0u);
|
||||
stats_.set_dumped_count(0u);
|
||||
stats_.set_trained_costtime(0u);
|
||||
stats_.set_trained_count(0u);
|
||||
|
||||
centroid_num_vec_.clear();
|
||||
cluster_class_.clear();
|
||||
converter_class_.clear();
|
||||
cluster_params_.clear();
|
||||
|
||||
labels_.clear();
|
||||
centroid_index_.reset();
|
||||
holder_.reset();
|
||||
converted_meta_ = meta_;
|
||||
converter_.reset();
|
||||
quantized_meta_ = meta_;
|
||||
quantizers_.clear();
|
||||
|
||||
error_ = false;
|
||||
err_code_ = 0;
|
||||
|
||||
thread_count_ = 0;
|
||||
sample_count_ = 0;
|
||||
cluster_auto_tuning_ = false;
|
||||
store_original_features_ = false;
|
||||
quantize_by_centroid_ = false;
|
||||
|
||||
LOG_INFO("End IVFBuilder::cleanup");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::train(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) {
|
||||
LOG_INFO("Begin IVFBuilder::train with holder");
|
||||
if (state_ != INITED) {
|
||||
LOG_ERROR("IVFBuilder train failed, wrong state=%d", state_);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
ailego::ElapsedTime timer;
|
||||
if (!holder || holder->count() == 0) {
|
||||
LOG_ERROR("Input holder is nullptr or empty while train index");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (!holder->is_matched(meta_)) {
|
||||
LOG_ERROR("Input holder doesn't match index meta while train index");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
if (converter_) {
|
||||
int ret = IndexConverter::TrainAndTransform(converter_, std::move(holder));
|
||||
ivf_check_with_msg(ret, "Failed to train or transform by converter %s",
|
||||
converter_->name().c_str());
|
||||
converted_meta_ = converter_->meta();
|
||||
holder = converter_->result();
|
||||
}
|
||||
|
||||
ailego::Params train_params;
|
||||
int ret = prepare_trainer_params(train_params);
|
||||
ivf_check_with_msg(ret, "Failed to prepare trainer params, ret=%d", ret);
|
||||
|
||||
IndexTrainer::Pointer trainer =
|
||||
IndexFactory::CreateTrainer("StratifiedClusterTrainer");
|
||||
ivf_assert_with_msg(trainer, IndexError_NoExist, "Failed to create trainer");
|
||||
|
||||
ret = trainer->init(converted_meta_, train_params);
|
||||
ivf_check_with_msg(ret, "Trainer init failed with ret %d", ret);
|
||||
|
||||
ret = trainer->train(std::move(threads), std::move(holder));
|
||||
ivf_check_with_msg(ret, "Trainer train failed with ret %d", ret);
|
||||
|
||||
ret = this->train(trainer);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
stats_.set_trained_costtime(timer.milli_seconds());
|
||||
|
||||
LOG_INFO("End IVFBuilder::train with holder");
|
||||
|
||||
state_ = TRAINED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::train(const IndexTrainer::Pointer &trainer) {
|
||||
LOG_DEBUG("Begin IVFBuilder::train by trainer");
|
||||
ailego::ElapsedTime timer;
|
||||
|
||||
if (state_ != INITED) {
|
||||
LOG_ERROR("IVFBuilder train failed, wrong state=%d", state_);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
if (!trainer) {
|
||||
LOG_ERROR("Input trainer is nullptr while train index");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
IndexCluster::CentroidList centroid_list;
|
||||
IndexBundle::Pointer boundle = trainer->indexes();
|
||||
int ret = IndexCluster::Deserialize(trainer->meta(), boundle, ¢roid_list);
|
||||
ivf_check_with_msg(ret, "Failed to deserialize index");
|
||||
|
||||
const IndexMeta &meta = trainer->meta();
|
||||
if (meta.data_type() != converted_meta_.data_type() ||
|
||||
meta.metric_name().compare(converted_meta_.metric_name()) != 0 ||
|
||||
meta.element_size() != converted_meta_.element_size()) {
|
||||
if (meta.converter_name() != converter_class_) {
|
||||
LOG_ERROR("Input trainer doesn't match index meta while train index");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
//! Create converter from trainer params
|
||||
LOG_INFO("Train IVFBuilder by trainer with converter");
|
||||
converter_ = CreateAndInitConverter(meta_, meta.converter_name(),
|
||||
meta.converter_params());
|
||||
ivf_assert(converter_, IndexError_Runtime);
|
||||
converted_meta_ = meta;
|
||||
}
|
||||
|
||||
centroid_index_ = std::make_shared<IVFCentroidIndex>();
|
||||
if (!centroid_index_) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
ret = centroid_index_->init(converted_meta_, params_);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
ret = centroid_index_->build(centroid_list);
|
||||
ivf_check_with_msg(ret, "Failed to build centroid index");
|
||||
|
||||
if (params_.has(PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_CLASS)) {
|
||||
//! Quantize the centroids for searcher
|
||||
searcher_centroid_index_ = std::make_shared<IVFCentroidIndex>();
|
||||
if (!searcher_centroid_index_) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
ailego::Params params;
|
||||
params_.get(PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_PARAMS, ¶ms);
|
||||
searcher_centroid_index_->set_quantizer(
|
||||
params_.get_as_string(PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_CLASS),
|
||||
params);
|
||||
ret = searcher_centroid_index_->init(converted_meta_, params_);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
ret = searcher_centroid_index_->build(centroid_list);
|
||||
ivf_check_with_msg(ret, "Failed to build centroid index");
|
||||
}
|
||||
|
||||
stats_.set_trained_costtime(timer.milli_seconds());
|
||||
|
||||
LOG_DEBUG("End IVFBuilder::train by trainer");
|
||||
|
||||
state_ = TRAINED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) {
|
||||
LOG_INFO("Begin IVFBuilder::build!");
|
||||
|
||||
if (state_ != TRAINED) {
|
||||
LOG_ERROR("Train the index first before build");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
if (!holder || holder->count() == 0) {
|
||||
LOG_ERROR("Input holder is nullptr or empty while building index");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!holder->is_matched(meta_)) {
|
||||
LOG_ERROR("Input holder doesn't match index meta while building index");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
|
||||
holder_ = std::make_shared<RandomAccessIndexHolder>(meta_);
|
||||
if (!holder_) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
if (holder->count() > 0) {
|
||||
holder_->reserve(holder->count());
|
||||
}
|
||||
for (auto iter = holder->create_iterator(); iter && iter->is_valid();
|
||||
iter->next()) {
|
||||
holder_->emplace(iter->key(), iter->data());
|
||||
}
|
||||
|
||||
// Holder is not needed, cleanup it.
|
||||
holder.reset();
|
||||
|
||||
IndexHolder::Pointer converted_holder = holder_;
|
||||
if (converter_) {
|
||||
int ret = converter_->transform(holder_);
|
||||
ivf_check_with_msg(ret, "Failed to transform by converter %s",
|
||||
converter_->name().c_str());
|
||||
converted_holder = converter_->result();
|
||||
}
|
||||
|
||||
labels_.resize(centroid_index_->centroids_count());
|
||||
int ret = this->build_label_index(threads.get(), converted_holder);
|
||||
ivf_check_with_msg(ret, "Failed to build index for %s",
|
||||
IndexError::What(ret));
|
||||
|
||||
ret = this->prepare_quantizer(threads.get());
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
stats_.set_built_costtime(timer.milli_seconds());
|
||||
|
||||
LOG_INFO("End IVFBuilder::build");
|
||||
|
||||
state_ = BUILT;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::dump(const IndexDumper::Pointer &dumper) {
|
||||
LOG_INFO("Begin IVFBuilder::dump");
|
||||
|
||||
if (state_ != BUILT) {
|
||||
LOG_ERROR("Build the index before dump QC Index");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
int ret = this->dump_index(dumper);
|
||||
ivf_check_with_msg(ret, "Failed to dump index with ret=%d", ret);
|
||||
|
||||
// the fitting function for the follow points: 1000000(0.02) 10000000(0.01)
|
||||
// 50000000(0.005) 100000000(0.001)
|
||||
float scan_ratio = -0.004 * std::log(holder_->count()) + 0.0751;
|
||||
scan_ratio = std::max(scan_ratio, 0.0001f);
|
||||
|
||||
// Set Searcher Params
|
||||
ailego::Params params;
|
||||
params.set(PARAM_IVF_SEARCHER_SCAN_RATIO, scan_ratio);
|
||||
meta_.set_searcher("IVFSearcher", 0, std::move(params));
|
||||
meta_.set_builder("IVFBuilder", 0, std::move(params_));
|
||||
|
||||
ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to serialize meta into dumper.");
|
||||
return ret;
|
||||
}
|
||||
|
||||
stats_.set_discarded_count(stats_.built_count() - stats_.dumped_count());
|
||||
stats_.set_dumped_costtime(timer.milli_seconds());
|
||||
|
||||
LOG_INFO("End IVFBuilder::dump");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::CheckAndUpdateMajorOrder(IndexMeta &meta) {
|
||||
const std::string &metric_name = meta.metric_name();
|
||||
auto metric = IndexFactory::CreateMetric(metric_name);
|
||||
if (!metric) {
|
||||
LOG_ERROR("CreateMetric %s failed", metric_name.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
int ret = metric->init(meta, meta.metric_params());
|
||||
ivf_check_with_msg(ret, "IndexMetric %s init failed", metric_name.c_str());
|
||||
|
||||
bool support_column_major = true;
|
||||
for (size_t m = 32; m != 0; m /= 2) {
|
||||
for (size_t n = m; n != 0; n /= 2) {
|
||||
if (metric->distance_matrix(m, n) == nullptr) {
|
||||
support_column_major = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!support_column_major) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
support_column_major &=
|
||||
meta.element_size() % IndexMeta::AlignSizeof(meta.data_type()) == 0;
|
||||
|
||||
if (meta.major_order() == IndexMeta::MO_UNDEFINED) {
|
||||
if (support_column_major && meta.dimension() <= 512) {
|
||||
meta.set_major_order(IndexMeta::MO_COLUMN);
|
||||
} else {
|
||||
meta.set_major_order(IndexMeta::MO_ROW);
|
||||
}
|
||||
} else {
|
||||
if (!support_column_major && meta.major_order() == IndexMeta::MO_COLUMN) {
|
||||
LOG_WARN(
|
||||
"Index Metric %s Unsupported "
|
||||
"Column Major Order",
|
||||
metric_name.c_str());
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
}
|
||||
|
||||
if (block_vector_count_ * quantized_meta_.element_size() % 32 != 0) {
|
||||
LOG_ERROR(
|
||||
"block_vector_count * quantized_element_size not align with 32 bytes.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::parse_centroids_num(const ailego::Params ¶ms) {
|
||||
std::string centroids_num =
|
||||
params.get_as_string(PARAM_IVF_BUILDER_CENTROID_COUNT);
|
||||
if (centroids_num.empty()) {
|
||||
LOG_ERROR("Param %s is required", PARAM_IVF_BUILDER_CENTROID_COUNT.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::vector<std::string> centroid_str_vec;
|
||||
ailego::StringHelper::Split(centroids_num, CENTROID_SEPERATOR,
|
||||
¢roid_str_vec);
|
||||
size_t level_cnt = centroid_str_vec.size();
|
||||
if ((level_cnt <= 0) || (level_cnt > 2)) {
|
||||
LOG_ERROR("Centroids level count must be [1,2]");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
for (size_t idx = 0; idx < level_cnt; ++idx) {
|
||||
uint32_t centroid_cnt = 0;
|
||||
if (!ailego::StringHelper::ToUint32(centroid_str_vec[idx], ¢roid_cnt)) {
|
||||
LOG_ERROR("Invalid centroids count %s", centroid_str_vec[idx].c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
centroid_num_vec_.push_back(centroid_cnt);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::parse_clustering_params(const ailego::Params ¶ms) {
|
||||
params.get(PARAM_IVF_BUILDER_CLUSTER_AUTO_TUNING, &cluster_auto_tuning_);
|
||||
|
||||
cluster_class_ = params.get_as_string(PARAM_IVF_BUILDER_CLUSTER_CLASS);
|
||||
if (cluster_class_.empty()) {
|
||||
// OptKmeansCluster does not support custom metric
|
||||
cluster_class_ = meta_.metric_name() == kMipsMetricName
|
||||
? "KmeansCluster"
|
||||
: "OptKmeansCluster";
|
||||
LOG_INFO("Using [%s] as default cluster class", cluster_class_.c_str());
|
||||
}
|
||||
for (size_t i = 1; i <= centroid_num_vec_.size(); ++i) {
|
||||
std::string level_params_key =
|
||||
PARAM_IVF_BUILDER_CLUSTER_PARAMS_IN_LEVEL_PREFIX + std::to_string(i);
|
||||
ailego::Params level_params;
|
||||
params.get<ailego::Params>(level_params_key, &level_params);
|
||||
cluster_params_.push_back(level_params);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::parse_general_params(const ailego::Params ¶ms) {
|
||||
thread_count_ = params.get_as_uint32(PARAM_IVF_BUILDER_THREAD_COUNT);
|
||||
sample_count_ = params.get_as_uint32(PARAM_IVF_BUILDER_TRAIN_SAMPLE_COUNT);
|
||||
sample_ratio_ = params.get_as_float(PARAM_IVF_BUILDER_TRAIN_SAMPLE_RATIO);
|
||||
|
||||
params.get(PARAM_IVF_BUILDER_QUANTIZE_BY_CENTROID, &quantize_by_centroid_);
|
||||
params.get(PARAM_IVF_BUILDER_STORE_ORIGINAL_FEATURES,
|
||||
&store_original_features_);
|
||||
|
||||
//! Prepare Converter for training
|
||||
if (meta_.metric_name() == kIPMetricName) {
|
||||
converter_class_ = kMipsConverterName;
|
||||
}
|
||||
params.get(PARAM_IVF_BUILDER_CONVERTER_CLASS, &converter_class_);
|
||||
if (!converter_class_.empty()) {
|
||||
ailego::Params converter_params;
|
||||
params_.get(PARAM_IVF_BUILDER_CONVERTER_PARAMS, &converter_params);
|
||||
converter_ =
|
||||
CreateAndInitConverter(meta_, converter_class_, converter_params);
|
||||
ivf_assert(converter_, IndexError_NoExist);
|
||||
}
|
||||
|
||||
params_.get(PARAM_IVF_BUILDER_BLOCK_VECTOR_COUNT, &block_vector_count_);
|
||||
if (block_vector_count_ == 0) {
|
||||
block_vector_count_ = kDefaultBlockCount;
|
||||
}
|
||||
if (block_vector_count_ > kDefaultBlockCount ||
|
||||
block_vector_count_ & (block_vector_count_ - 1)) {
|
||||
LOG_ERROR("block_vector_count only can be [1|2|4|8|16|32].");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (block_vector_count_ * meta_.element_size() % 32 != 0) {
|
||||
LOG_ERROR("block_vector_count * element_size not align with 32 bytes.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::prepare_trainer_params(ailego::Params ¶ms) {
|
||||
params.set(STRATIFIED_TRAINER_SAMPLE_COUNT, sample_count_);
|
||||
params.set(STRATIFIED_TRAINER_SAMPLE_RATIO, sample_ratio_);
|
||||
params.set(STRATIFIED_TRAINER_THREAD_COUNT, thread_count_);
|
||||
params.set(STRATIFIED_TRAINER_AUTOAUNE, cluster_auto_tuning_);
|
||||
if (centroid_num_vec_.empty()) {
|
||||
LOG_ERROR("Centroids no specified.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
std::string cluster_count = std::to_string(centroid_num_vec_[0]);
|
||||
if (centroid_num_vec_.size() > 1) {
|
||||
cluster_count +=
|
||||
(CENTROID_SEPERATOR + std::to_string(centroid_num_vec_[1]));
|
||||
}
|
||||
params.set(STRATIFIED_TRAINER_CLUSTER_COUNT, cluster_count);
|
||||
|
||||
for (size_t i = 1; i <= cluster_params_.size(); ++i) {
|
||||
std::string level_params_key =
|
||||
STRATIFIED_TRAINER_PARAMS_IN_LEVEL_PREFIX + std::to_string(i);
|
||||
params.set(level_params_key, cluster_params_[i - 1]);
|
||||
}
|
||||
params.set(STRATIFIED_TRAINER_CLASS_NAME, cluster_class_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::build_label_index(IndexThreads *threads,
|
||||
const IndexHolder::Pointer &holder) {
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
size_t id = 0UL;
|
||||
AILEGO_DEFER([&]() {
|
||||
task_group->wait_finish();
|
||||
stats_.set_built_count(id);
|
||||
LOG_INFO("Finished building, total=%zu", id);
|
||||
});
|
||||
|
||||
size_t elem_size = holder->element_size();
|
||||
std::shared_ptr<VectorList> vectors = std::make_shared<VectorList>();
|
||||
ivf_assert(vectors, IndexError_NoMemory);
|
||||
for (; iter && iter->is_valid(); iter->next()) {
|
||||
ivf_assert(!error_, err_code_);
|
||||
vectors->emplace_back(iter->data(), elem_size, id);
|
||||
id++;
|
||||
if (vectors->size() == kBatchSize || id == holder_->count()) {
|
||||
auto task = ailego::Closure ::New(const_cast<IVFBuilder *>(this),
|
||||
&IVFBuilder::label, vectors);
|
||||
task_group->submit(std::move(task));
|
||||
vectors = std::make_shared<VectorList>();
|
||||
ivf_assert(vectors, IndexError_NoMemory);
|
||||
vectors->reserve(kBatchSize);
|
||||
}
|
||||
if (!(id & 0xFFFFF)) {
|
||||
LOG_INFO("Current built count:%zu", id);
|
||||
}
|
||||
}
|
||||
ailego_assert_with(vectors->size() == 0, "invalid size");
|
||||
|
||||
return err_code_;
|
||||
}
|
||||
|
||||
int IVFBuilder::dump_index(const IndexDumper::Pointer &dumper) {
|
||||
int ret = CheckAndUpdateMajorOrder(quantized_meta_);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
IVFDumper::Pointer ivf_dumper = std::make_shared<IVFDumper>(
|
||||
quantized_meta_, dumper, centroid_index_->centroids_count(),
|
||||
block_vector_count_);
|
||||
if (!ivf_dumper) {
|
||||
LOG_ERROR("Alloc IVFDumper failed");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
|
||||
//! Dump inverted vectors
|
||||
std::vector<uint32_t> dumped_ids;
|
||||
std::function<void(uint32_t)> record_dumped_id = [&](uint32_t) {};
|
||||
if (store_original_features_) {
|
||||
dumped_ids.reserve(holder_->count());
|
||||
record_dumped_id = [&](uint32_t id) { dumped_ids.emplace_back(id); };
|
||||
}
|
||||
if (quantizers_.size() == 0) {
|
||||
//! No quantizer for inverted vectors
|
||||
for (size_t i = 0; i < centroid_index_->centroids_count(); ++i) {
|
||||
ailego_assert_with(i < labels_.size(), "Index Overflow");
|
||||
for (size_t j = 0; j < labels_[i].size(); ++j) {
|
||||
auto id = labels_[i][j];
|
||||
record_dumped_id(id);
|
||||
ret = ivf_dumper->dump_inverted_vector(i, holder_->key(id),
|
||||
holder_->element(id));
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (size_t i = 0; i < centroid_index_->centroids_count(); ++i) {
|
||||
ailego_assert_with(i < labels_.size(), "Index Overflow");
|
||||
auto holder =
|
||||
std::make_shared<LabelFilteredIndexHolder>(holder_, labels_[i]);
|
||||
if (!holder) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
auto quantizer = quantize_by_centroid_ ? quantizers_[i] : quantizers_[0];
|
||||
ret = quantizer->transform(holder);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
auto iter = quantizer->result()->create_iterator();
|
||||
for (; iter->is_valid(); iter->next()) {
|
||||
uint32_t id = iter->key();
|
||||
record_dumped_id(id);
|
||||
ret =
|
||||
ivf_dumper->dump_inverted_vector(i, holder_->key(id), iter->data());
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret = ivf_dumper->dump_inverted_vector_finished();
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
ret = ivf_dumper->dump_quantizer_params(quantizers_);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
auto centroid_index =
|
||||
searcher_centroid_index_ ? searcher_centroid_index_ : centroid_index_;
|
||||
ret = ivf_dumper->dump_centroid_index(centroid_index->data(),
|
||||
centroid_index->size());
|
||||
ivf_check_with_msg(ret, "Failed to dump CentroidIndex");
|
||||
|
||||
if (store_original_features_) {
|
||||
for (size_t i = 0; i < dumped_ids.size(); ++i) {
|
||||
ret = ivf_dumper->dump_original_vector(holder_->element(dumped_ids[i]),
|
||||
holder_->element_size());
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
}
|
||||
|
||||
stats_.set_dumped_count(stats_.dumped_count() + ivf_dumper->dumped_count());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFBuilder::prepare_quantizer(IndexThreads *threads) {
|
||||
std::string quantizer_name;
|
||||
params_.get(PARAM_IVF_BUILDER_QUANTIZER_CLASS, &quantizer_name);
|
||||
if (quantizer_name.empty()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Prepare Quantizers for inverted index
|
||||
ailego::Params quantizer_params;
|
||||
params_.get(PARAM_IVF_BUILDER_QUANTIZER_PARAMS, &quantizer_params);
|
||||
if (((quantizer_name != kInt8QuantizerName &&
|
||||
quantizer_name != kInt4QuantizerName) ||
|
||||
meta_.metric_name() != kIPMetricName) &&
|
||||
quantize_by_centroid_) {
|
||||
LOG_WARN("%s is supported in InnerProduct only",
|
||||
PARAM_IVF_BUILDER_QUANTIZE_BY_CENTROID.c_str());
|
||||
quantize_by_centroid_ = false;
|
||||
}
|
||||
if (quantizer_name == kInt4QuantizerName && meta_.dimension() & 0x1) {
|
||||
LOG_ERROR("Unsupport quantizer=%s for dim=%u", kInt4QuantizerName,
|
||||
meta_.dimension());
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
int ret = 0;
|
||||
auto create_and_init_quantizer = [&]() {
|
||||
auto quantizer = IndexFactory::CreateConverter(quantizer_name);
|
||||
if (!quantizer) {
|
||||
LOG_ERROR("Failed to create converter %s", quantizer_name.c_str());
|
||||
ret = IndexError_NoExist;
|
||||
return IndexConverter::Pointer();
|
||||
}
|
||||
ret = quantizer->init(meta_, quantizer_params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize converter %s for %s",
|
||||
quantizer_name.c_str(), IndexError::What(ret));
|
||||
return IndexConverter::Pointer();
|
||||
}
|
||||
return quantizer;
|
||||
};
|
||||
for (size_t i = 0; i < centroid_index_->centroids_count(); ++i) {
|
||||
quantizers_.emplace_back(create_and_init_quantizer());
|
||||
ivf_check_error_code(ret);
|
||||
if (!quantize_by_centroid_) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
//! Train the quantizers
|
||||
auto train_data = [&](size_t i) {
|
||||
IndexHolder::Pointer holder = holder_;
|
||||
size_t idx = 0;
|
||||
if (quantize_by_centroid_) {
|
||||
holder = std::make_shared<LabelFilteredIndexHolder>(holder_, labels_[i]);
|
||||
if (!holder && !error_.exchange(true)) {
|
||||
err_code_ = IndexError_NoMemory;
|
||||
return;
|
||||
}
|
||||
idx = i;
|
||||
}
|
||||
if (holder->count() == 0) {
|
||||
return;
|
||||
}
|
||||
ret = quantizers_[idx]->train(holder);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to train converter %s for %s", quantizer_name.c_str(),
|
||||
IndexError::What(ret));
|
||||
if (!error_.exchange(true)) {
|
||||
err_code_ = IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < quantizers_.size(); ++i) {
|
||||
if (error_) {
|
||||
task_group->wait_finish();
|
||||
return err_code_;
|
||||
}
|
||||
task_group->submit(ailego::Closure ::New(train_data, i));
|
||||
}
|
||||
|
||||
task_group->wait_finish();
|
||||
if (quantizers_.size() > 0) {
|
||||
quantized_meta_ = quantizers_[0]->meta();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_BUILDER(IVFBuilder);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,316 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "framework/index_builder.h"
|
||||
#include "framework/index_meta.h"
|
||||
#include "ivf_centroid_index.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! IVF Builder
|
||||
*/
|
||||
class IVFBuilder : public IndexBuilder {
|
||||
public:
|
||||
//! Constructor
|
||||
IVFBuilder();
|
||||
|
||||
//! Destructor
|
||||
~IVFBuilder();
|
||||
|
||||
//! Disable them
|
||||
IVFBuilder(const IVFBuilder &) = delete;
|
||||
IVFBuilder &operator=(const IVFBuilder &) = delete;
|
||||
|
||||
public:
|
||||
//! Initialize the builder
|
||||
virtual int init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup the builder
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(const IndexTrainer::Pointer &trainer) override;
|
||||
|
||||
//! Build the index
|
||||
virtual int build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Dump index into file system
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
IVFCentroidIndex::Pointer centroid_index() const {
|
||||
return centroid_index_;
|
||||
}
|
||||
|
||||
public:
|
||||
/*! Random Access Index Holder
|
||||
*/
|
||||
class RandomAccessIndexHolder : public IndexHolder {
|
||||
public:
|
||||
//! Index Holder Iterator Pointer
|
||||
typedef std::shared_ptr<RandomAccessIndexHolder> Pointer;
|
||||
|
||||
/*! Random Access Index Holder Iterator
|
||||
*/
|
||||
class Iterator : public IndexHolder::Iterator {
|
||||
public:
|
||||
//! Index Holder Iterator Pointer
|
||||
typedef std::unique_ptr<Iterator> Pointer;
|
||||
|
||||
//! Constructor
|
||||
Iterator(RandomAccessIndexHolder *owner) : holder_(owner) {}
|
||||
|
||||
//! Destructor
|
||||
virtual ~Iterator(void) {}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
virtual const void *data(void) const override {
|
||||
return holder_->element(id_);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
return id_ < holder_->count();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
return holder_->key(id_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
++id_;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
RandomAccessIndexHolder *holder_{nullptr};
|
||||
uint32_t id_{0};
|
||||
};
|
||||
|
||||
//! Constructor
|
||||
RandomAccessIndexHolder(const IndexMeta &meta)
|
||||
: features_(std::make_shared<CompactIndexFeatures>(meta)) {}
|
||||
|
||||
//! Retrieve count of elements in holder (-1 indicates unknown)
|
||||
virtual size_t count(void) const override {
|
||||
return features_->count();
|
||||
}
|
||||
|
||||
//! Retrieve dimension
|
||||
virtual size_t dimension(void) const override {
|
||||
return features_->dimension();
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
virtual IndexMeta::DataType data_type(void) const override {
|
||||
return features_->data_type();
|
||||
}
|
||||
|
||||
//! Retrieve element size in bytes
|
||||
virtual size_t element_size(void) const override {
|
||||
return features_->element_size();
|
||||
}
|
||||
|
||||
//! Retrieve if it can multi-pass
|
||||
virtual bool multipass(void) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
|
||||
return IndexHolder::Iterator::Pointer(
|
||||
new RandomAccessIndexHolder::Iterator(this));
|
||||
}
|
||||
|
||||
void reserve(size_t elems) {
|
||||
features_->reserve(elems);
|
||||
keys_.reserve(elems);
|
||||
}
|
||||
|
||||
//! Append an element into holder
|
||||
void emplace(uint64_t pkey, const void *vec) {
|
||||
features_->emplace(vec);
|
||||
keys_.emplace_back(pkey);
|
||||
}
|
||||
|
||||
//! Retrieve feature via local id
|
||||
const void *element(size_t id) const {
|
||||
return features_->element(id);
|
||||
}
|
||||
|
||||
//! Retrieve key via local id
|
||||
uint64_t key(size_t id) const {
|
||||
ailego_assert_with(id < keys_.size(), "Index Overflow");
|
||||
return keys_[id];
|
||||
}
|
||||
|
||||
private:
|
||||
//! Disable them
|
||||
RandomAccessIndexHolder(void) = delete;
|
||||
|
||||
//! Members
|
||||
CompactIndexFeatures::Pointer features_{};
|
||||
std::vector<uint64_t> keys_{};
|
||||
};
|
||||
|
||||
private:
|
||||
/*! Wrapper of feature
|
||||
*/
|
||||
class Vector {
|
||||
public:
|
||||
typedef std::shared_ptr<Vector> Pointer;
|
||||
|
||||
Vector(const void *vec, size_t len, uint32_t idx)
|
||||
: vec_(reinterpret_cast<const char *>(vec), len), id_{idx} {}
|
||||
|
||||
const void *data() const {
|
||||
return vec_.data();
|
||||
}
|
||||
|
||||
size_t size() const {
|
||||
return vec_.size();
|
||||
}
|
||||
|
||||
uint32_t id(void) const {
|
||||
return id_;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string vec_{};
|
||||
uint32_t id_{0u};
|
||||
};
|
||||
|
||||
using VectorList = std::vector<Vector>;
|
||||
|
||||
//! Check MajorOrder in meta, and update the major order if needed
|
||||
int CheckAndUpdateMajorOrder(IndexMeta &meta);
|
||||
|
||||
//! Parse params
|
||||
int parse_centroids_num(const ailego::Params ¶ms);
|
||||
int parse_clustering_params(const ailego::Params ¶ms);
|
||||
int parse_general_params(const ailego::Params ¶ms);
|
||||
|
||||
//! Prepare params for trainer
|
||||
int prepare_trainer_params(ailego::Params ¶ms);
|
||||
|
||||
//! Build the index
|
||||
int build_label_index(IndexThreads *threads,
|
||||
const IndexHolder::Pointer &holder);
|
||||
|
||||
//! Dump the index to dumper
|
||||
int dump_index(const IndexDumper::Pointer &dumper);
|
||||
|
||||
//! Prepare the quantizer for inverted index
|
||||
int prepare_quantizer(IndexThreads *threads);
|
||||
|
||||
//! Quantize the centrods list
|
||||
int quantize_centroids();
|
||||
|
||||
//! Create converter and init with params
|
||||
static IndexConverter::Pointer CreateAndInitConverter(
|
||||
const IndexMeta &meta, const std::string &name,
|
||||
const ailego::Params ¶ms) {
|
||||
auto converter = IndexFactory::CreateConverter(name);
|
||||
if (!converter) {
|
||||
LOG_ERROR("Failed to create converter %s", name.c_str());
|
||||
return IndexConverter::Pointer();
|
||||
}
|
||||
int ret = converter->init(meta, params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize converter %s for %s", name.c_str(),
|
||||
IndexError::What(ret));
|
||||
return IndexConverter::Pointer();
|
||||
}
|
||||
return converter;
|
||||
}
|
||||
|
||||
//! Select the nearest centroid id for the vector
|
||||
void label(const std::shared_ptr<VectorList> &vecs) {
|
||||
for (size_t i = 0; i < vecs->size(); ++i) {
|
||||
auto &vec = (*vecs)[i];
|
||||
|
||||
uint32_t centroid_idx =
|
||||
centroid_index_->search_nearest_centroid(vec.data(), vec.size());
|
||||
if (centroid_idx == IVFCentroidIndex::kInvalidID) {
|
||||
LOG_ERROR("Failed to search nearest centroid in CentroidIndex");
|
||||
if (!error_.exchange(true)) {
|
||||
err_code_ = IndexError_Runtime;
|
||||
}
|
||||
return;
|
||||
}
|
||||
ailego_assert_with(centroid_idx < labels_.size(), "Index Overflow");
|
||||
mutex_.lock();
|
||||
labels_[centroid_idx].emplace_back(vec.id());
|
||||
mutex_.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
//! Constants
|
||||
static constexpr size_t kThreadPoolQueueSize = 300u;
|
||||
static constexpr size_t kBatchSize = 10u;
|
||||
static constexpr size_t kDefaultBlockCount = 32u;
|
||||
|
||||
enum BuilderState { INIT = 0, INITED = 1, TRAINED = 2, BUILT = 3 };
|
||||
|
||||
//! Members
|
||||
BuilderState state_{INIT};
|
||||
Stats stats_{};
|
||||
ailego::Params params_{};
|
||||
IndexMeta meta_{};
|
||||
|
||||
std::vector<uint32_t> centroid_num_vec_{};
|
||||
std::string cluster_class_{};
|
||||
std::string converter_class_{};
|
||||
std::vector<ailego::Params> cluster_params_{};
|
||||
|
||||
std::vector<std::vector<uint32_t>> labels_{};
|
||||
std::mutex mutex_{};
|
||||
IVFCentroidIndex::Pointer centroid_index_{};
|
||||
IVFCentroidIndex::Pointer searcher_centroid_index_{};
|
||||
RandomAccessIndexHolder::Pointer holder_{};
|
||||
IndexMeta converted_meta_{};
|
||||
IndexConverter::Pointer converter_{};
|
||||
IndexMeta quantized_meta_{};
|
||||
std::vector<IndexConverter::Pointer> quantizers_{};
|
||||
|
||||
std::atomic_bool error_{false};
|
||||
int err_code_{0};
|
||||
|
||||
uint32_t thread_count_{0};
|
||||
uint32_t sample_count_{0};
|
||||
float sample_ratio_{0.0};
|
||||
uint32_t block_vector_count_{kDefaultBlockCount};
|
||||
bool cluster_auto_tuning_{false};
|
||||
bool store_original_features_{false};
|
||||
bool quantize_by_centroid_{false};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,621 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "ivf_centroid_index.h"
|
||||
#include "framework/index_framework.h"
|
||||
#include "metric/metric_params.h"
|
||||
#include "quantizer/quantizer_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Fake Trainer to supply centroids in bundle
|
||||
*/
|
||||
class FakeClusterTrainer : public IndexTrainer {
|
||||
public:
|
||||
//! Constructor
|
||||
FakeClusterTrainer(const IndexMeta &imeta, const IndexBundle::Pointer &bundle)
|
||||
: meta_(imeta), bundle_(bundle) {}
|
||||
|
||||
//! Destructor
|
||||
~FakeClusterTrainer(void) {}
|
||||
|
||||
protected:
|
||||
//! Initialize Trainer
|
||||
virtual int init(const IndexMeta &, const ailego::Params &) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cleanup Trainer
|
||||
virtual int cleanup(void) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Train the data
|
||||
virtual int train(IndexHolder::Pointer) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Train the data
|
||||
virtual int train(IndexThreads::Pointer, IndexHolder::Pointer) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Load index from file path or dir
|
||||
virtual int load(IndexStorage::Pointer) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Dump index into file path or dir
|
||||
virtual int dump(const IndexDumper::Pointer &) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve Index Meta
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const IndexTrainer::Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve the output indexes
|
||||
virtual IndexBundle::Pointer indexes(void) const override {
|
||||
return bundle_;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
IndexMeta meta_{};
|
||||
Stats stats_{};
|
||||
IndexBundle::Pointer bundle_{};
|
||||
};
|
||||
|
||||
/*! Int8QuantizerReformer for InnerProduct Measure
|
||||
*/
|
||||
class Int8QuantizerReformer4IP : public IndexReformer {
|
||||
public:
|
||||
//! Initialize Reformer
|
||||
virtual int init(const ailego::Params &) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cleanup Reformer
|
||||
virtual int cleanup(void) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Load index from container
|
||||
virtual int load(IndexStorage::Pointer) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Unload index
|
||||
virtual int unload(void) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Transform query
|
||||
virtual int transform(const void * /*query*/,
|
||||
const IndexQueryMeta & /*qmeta*/, std::string * /*out*/,
|
||||
IndexQueryMeta * /*ometa*/) const override {
|
||||
#if 0
|
||||
size_t dim = qmeta.dimension();
|
||||
out->resize(IndexMeta::ElementSizeof(
|
||||
IndexMeta::DataType::DT_INT8, dim));
|
||||
ometa->set_meta(IndexMeta::DataType::DT_INT8, dim);
|
||||
const float *ivec = reinterpret_cast<const float *>(query);
|
||||
int8_t *ovec = reinterpret_cast<int8_t *>(&(*out)[0]);
|
||||
float abs_max = 0.0f;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
auto abs = std::abs(ivec[i]);
|
||||
if (abs > abs_max) {
|
||||
abs_max = abs;
|
||||
}
|
||||
}
|
||||
if (abs_max > 0.0f) {
|
||||
float scale = 127 / abs_max;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
ovec[i] = static_cast<int8_t>(std::round(ivec[i] * scale));
|
||||
}
|
||||
} else {
|
||||
std::fill(ovec, ovec + dim, static_cast<int8_t>(1));
|
||||
}
|
||||
return 0;
|
||||
#else
|
||||
return IndexError_NotImplemented;
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Transform queries
|
||||
virtual int transform(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, std::string *oquery,
|
||||
IndexQueryMeta *ometa) const override {
|
||||
size_t dim = qmeta.dimension();
|
||||
oquery->resize(count *
|
||||
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT8, dim));
|
||||
ometa->set_meta(IndexMeta::DataType::DT_INT8, dim);
|
||||
const float *ivec = reinterpret_cast<const float *>(query);
|
||||
int8_t *ovec = reinterpret_cast<int8_t *>(&(*oquery)[0]);
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
float abs_max = 0.0f;
|
||||
const float *in = &ivec[q * dim];
|
||||
int8_t *out = &ovec[q * dim];
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
auto abs = std::abs(in[i]);
|
||||
if (abs > abs_max) {
|
||||
abs_max = abs;
|
||||
}
|
||||
}
|
||||
if (abs_max > 0.0f) {
|
||||
float scale = 127 / abs_max;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = static_cast<int8_t>(std::round(in[i] * scale));
|
||||
}
|
||||
} else {
|
||||
std::fill(out, out + dim, static_cast<int8_t>(1));
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Normalize results
|
||||
virtual int normalize(const void * /*query*/,
|
||||
const IndexQueryMeta & /*qmeta*/,
|
||||
IndexDocumentList & /*result*/) const override {
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
/*! Int4QuantizerReformer for InnerProduct Metric
|
||||
*/
|
||||
class Int4QuantizerReformer4IP : public IndexReformer {
|
||||
public:
|
||||
//! Initialize Reformer
|
||||
virtual int init(const ailego::Params &) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cleanup Reformer
|
||||
virtual int cleanup(void) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Load index from container
|
||||
virtual int load(IndexStorage::Pointer) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Unload index
|
||||
virtual int unload(void) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Transform query
|
||||
virtual int transform(const void * /*query*/,
|
||||
const IndexQueryMeta & /*qmeta*/, std::string * /*out*/,
|
||||
IndexQueryMeta * /*ometa*/) const override {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Transform queries
|
||||
virtual int transform(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, std::string *oquery,
|
||||
IndexQueryMeta *ometa) const override {
|
||||
if (qmeta.dimension() & 0x1) {
|
||||
LOG_ERROR("Unsuuport dim=%u for transform", qmeta.dimension());
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
size_t dim = qmeta.dimension();
|
||||
oquery->resize(count *
|
||||
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT4, dim));
|
||||
ometa->set_meta(IndexMeta::DataType::DT_INT4, dim);
|
||||
const float *ivec = reinterpret_cast<const float *>(query);
|
||||
uint8_t *ovec = reinterpret_cast<uint8_t *>(&(*oquery)[0]);
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
float abs_max = 0.0f;
|
||||
float max = -std::numeric_limits<float>::max();
|
||||
const float *in = &ivec[q * dim];
|
||||
uint8_t *out = &ovec[q * dim / 2];
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
float abs = std::abs(in[i]);
|
||||
abs_max = std::max(abs_max, abs);
|
||||
max = std::max(max, in[i]);
|
||||
}
|
||||
if (abs_max > 0.0f) {
|
||||
float scale = ((7 * abs_max > 8 * max) ? 8 : 7) / abs_max;
|
||||
for (size_t i = 0; i < dim; i += 2) {
|
||||
auto v1 = static_cast<int8_t>(std::round(in[i] * scale));
|
||||
auto v2 = static_cast<int8_t>(std::round(in[i + 1] * scale));
|
||||
out[i / 2] = (static_cast<uint8_t>(v1) << 4) |
|
||||
(static_cast<uint8_t>(v2) & 0xF);
|
||||
}
|
||||
} else {
|
||||
std::fill(out, out + dim / 2, static_cast<uint8_t>(9));
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Normalize results
|
||||
virtual int normalize(const void * /*query*/,
|
||||
const IndexQueryMeta & /*qmeta*/,
|
||||
IndexDocumentList & /*result*/) const override {
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
int IVFCentroidIndex::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
meta_ = meta;
|
||||
|
||||
params.get(PARAM_IVF_BUILDER_OPTIMIZER_CLASS, &builder_class_);
|
||||
params.get(PARAM_IVF_BUILDER_OPTIMIZER_PARAMS, &builder_params_);
|
||||
params.get(PARAM_IVF_SEARCHER_OPTIMIZER, &searcher_class_);
|
||||
params.get(PARAM_IVF_SEARCHER_OPTIMIZER_PARAMS, &searcher_params_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFCentroidIndex::search(const void *query, const IndexQueryMeta &qmeta,
|
||||
size_t count,
|
||||
IndexSearcher::Context::Pointer &ctx) {
|
||||
int ret = 0;
|
||||
|
||||
if (reformer_) {
|
||||
std::string buffer;
|
||||
IndexQueryMeta ometa;
|
||||
ret = reformer_->transform(query, qmeta, count, &buffer, &ometa);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to transform querys by reformer");
|
||||
return ret;
|
||||
}
|
||||
ret = searcher_->search_impl(buffer.data(), ometa, count, ctx);
|
||||
} else {
|
||||
ret = searcher_->search_impl(query, qmeta, count, ctx);
|
||||
}
|
||||
|
||||
ivf_check_with_msg(ret, "Failed to search in centroid index for %s",
|
||||
IndexError::What(ret));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t IVFCentroidIndex::search_nearest_centroid(const void *query,
|
||||
size_t len) {
|
||||
//! Called in building index precedure, so transform the query is needless
|
||||
if (len != meta_.element_size()) {
|
||||
LOG_ERROR("Invalid query size actual: %zu, expected: %u", len,
|
||||
meta_.element_size());
|
||||
return kInvalidID;
|
||||
}
|
||||
|
||||
thread_local IndexSearcher::Context::Pointer context(
|
||||
searcher_->create_context());
|
||||
context->set_topk(1);
|
||||
|
||||
IndexQueryMeta qmeta(meta_.data_type(), meta_.dimension());
|
||||
int ret = searcher_->search_impl(query, qmeta, context);
|
||||
if (ret != 0 || context->result().empty()) {
|
||||
LOG_ERROR("Failed to search nearest centroid, with ret %d", ret);
|
||||
return kInvalidID;
|
||||
}
|
||||
|
||||
return static_cast<uint32_t>(context->result()[0].key());
|
||||
}
|
||||
|
||||
uint32_t IVFCentroidIndex::transform_and_search_nearest_centroid(
|
||||
const void *record, const IndexQueryMeta &rmeta,
|
||||
IndexSearcher::Context::Pointer &ctx) const {
|
||||
int ret = 0;
|
||||
if (reformer_) {
|
||||
std::string buffer;
|
||||
IndexQueryMeta ometa;
|
||||
ret = reformer_->convert(record, rmeta, &buffer, &ometa);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to transform querys by reformer");
|
||||
return kInvalidID;
|
||||
}
|
||||
ret = searcher_->search_impl(buffer.data(), ometa, ctx);
|
||||
} else {
|
||||
ret = searcher_->search_impl(record, rmeta, ctx);
|
||||
}
|
||||
if (ret != 0 || ctx->result().empty()) {
|
||||
LOG_ERROR("Failed to search in centroid index for %s",
|
||||
IndexError::What(ret));
|
||||
return kInvalidID;
|
||||
}
|
||||
|
||||
return static_cast<uint32_t>(ctx->result()[0].key());
|
||||
}
|
||||
|
||||
IndexHolder::Pointer IVFCentroidIndex::quantize_holder(
|
||||
const IndexHolder::Pointer &holder) {
|
||||
auto input = holder;
|
||||
if (meta_.reformer_name() == kMipsReformerName &&
|
||||
meta_.metric_name() == kL2MetricName &&
|
||||
(quantizer_class_ == kInt8QuantizerName ||
|
||||
quantizer_class_ == kInt4QuantizerName)) {
|
||||
//! Reverse for Mips if do convert by integer quantizer
|
||||
auto reverse = IndexFactory::CreateConverter(kMipsRevConverterName);
|
||||
if (!reverse) {
|
||||
LOG_ERROR("Failed to create converter %s", kMipsRevConverterName);
|
||||
return nullptr;
|
||||
}
|
||||
ailego::Params params;
|
||||
auto p = meta_.reformer_params();
|
||||
params.set(MIPS_REVERSE_CONVERTER_M_VALUE,
|
||||
p.get_as_uint32(MIPS_REFORMER_M_VALUE));
|
||||
params.set(MIPS_REVERSE_CONVERTER_U_VALUE,
|
||||
p.get_as_float(MIPS_REFORMER_U_VALUE));
|
||||
params.set(MIPS_REVERSE_CONVERTER_L2_NORM,
|
||||
p.get_as_uint32(MIPS_REFORMER_L2_NORM));
|
||||
params.set(MIPS_REVERSE_CONVERTER_FORCED_SINGLE_FLOAT,
|
||||
p.get_as_float(MIPS_REFORMER_FORCED_HALF_FLOAT));
|
||||
int ret = reverse->init(meta_, params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Fail to init converter %s", kMipsRevConverterName);
|
||||
return nullptr;
|
||||
}
|
||||
ret = IndexConverter::TrainAndTransform(reverse, holder);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Fail to transform converter %s", kMipsRevConverterName);
|
||||
return nullptr;
|
||||
}
|
||||
input = reverse->result();
|
||||
meta_ = reverse->meta();
|
||||
meta_.set_metric(kIPMetricName, 0, ailego::Params());
|
||||
meta_.set_reformer("", 0, ailego::Params());
|
||||
}
|
||||
|
||||
auto converter = IndexFactory::CreateConverter(quantizer_class_);
|
||||
if (!converter) {
|
||||
LOG_ERROR("Failed to create converter %s", quantizer_class_.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
int ret = converter->init(meta_, quantizer_params_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Fail to init converter %s", quantizer_class_.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ret = IndexConverter::TrainAndTransform(converter, input);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Fail to tranform converter %s", quantizer_class_.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
meta_ = converter->meta();
|
||||
return converter->result();
|
||||
}
|
||||
|
||||
int IVFCentroidIndex::build_index(
|
||||
const IndexCluster::CentroidList ¢roid_list,
|
||||
const IndexDumper::Pointer &dumper) {
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder(builder_class_);
|
||||
if (!builder) {
|
||||
LOG_ERROR("Failed to create builder %s", builder_class_.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
IndexHolder::Pointer holder =
|
||||
std::make_shared<CentroidsIndexHolder>(meta_, centroid_list);
|
||||
if (!holder) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
if (holder->count() == 0) {
|
||||
LOG_ERROR("No centroids to build");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
centroids_count_ = holder->count();
|
||||
|
||||
//! Set default params if not given
|
||||
auto count = std::to_string(
|
||||
static_cast<size_t>(std::ceil(std::sqrt(centroids_count_ / 10.0))));
|
||||
// if (IsHcBuilder(builder_class_) &&
|
||||
// !builder_params_.has(hc::PARAM_HC_BUILDER_CENTROID_COUNT)) {
|
||||
// builder_params_.set(hc::PARAM_HC_BUILDER_CENTROID_COUNT, count);
|
||||
// } else if (builder_class_ == "GcBuilder" &&
|
||||
// !builder_params_.has(hc::PARAM_GC_BUILDER_CENTROID_COUNT)) {
|
||||
// builder_params_.set(hc::PARAM_GC_BUILDER_CENTROID_COUNT, count);
|
||||
// }
|
||||
if (!quantizer_class_.empty()) {
|
||||
holder = this->quantize_holder(holder);
|
||||
if (!holder) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
|
||||
const auto name = builder_class_.c_str();
|
||||
int ret = builder->init(meta_, builder_params_);
|
||||
ivf_check_with_msg(ret, "%s init failed, ret=%d", name, ret);
|
||||
|
||||
// if (IsHcBuilder(builder_class_) && quantizer_class_.empty()) {
|
||||
// auto trainer = this->prepare_trainer(centroid_list);
|
||||
// ret = trainer ? builder->train(trainer) : builder->train(holder);
|
||||
// } else {
|
||||
// ret = builder->train(holder);
|
||||
// }
|
||||
|
||||
ret = builder->train(holder);
|
||||
ivf_check_with_msg(ret, "%s train failed, ret=%d", name, ret);
|
||||
|
||||
ret = builder->build(holder);
|
||||
ivf_check_with_msg(ret, "%s build failed, ret=%d", name, ret);
|
||||
|
||||
ret = builder->dump(dumper);
|
||||
ivf_check_with_msg(ret, "%s dump failed, ret=%d", name, ret);
|
||||
|
||||
ret = dumper->close();
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFCentroidIndex::build(const IndexCluster::CentroidList ¢roid_list) {
|
||||
index_building_ = true;
|
||||
//! Build and dump the index
|
||||
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
|
||||
if (!dumper) {
|
||||
LOG_ERROR("Failed to create MemoryDumper");
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
path_ = IVFUtility::GenerateRandomPath(kTempralPathPrefix);
|
||||
int ret = dumper->create(path_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexDumper create path %s failed", path_.c_str());
|
||||
return ret;
|
||||
}
|
||||
ret = this->build_index(centroid_list, dumper);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
auto rope = IndexMemory::Instance()->open(path_);
|
||||
if (!rope) {
|
||||
LOG_ERROR("Open memory path %s failed.", path_.c_str());
|
||||
return ret;
|
||||
}
|
||||
if (rope->count() != 1) {
|
||||
LOG_ERROR("Graph Rope block count not equal with 1.");
|
||||
return ret;
|
||||
}
|
||||
(*rope)[0].read(0, &data_, 0);
|
||||
size_ = (*rope)[0].size();
|
||||
|
||||
//! Load the index
|
||||
IndexStorage::Pointer container =
|
||||
IndexFactory::CreateStorage("MemoryReadStorage");
|
||||
if (!container) {
|
||||
LOG_ERROR("Failed to create MemoryReadStorage");
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
ret = container->init(ailego::Params());
|
||||
ivf_check_with_msg(ret, "Failed to initialize MemoryReadStorage for %s",
|
||||
IndexError::What(ret));
|
||||
ret = container->open(path_, false);
|
||||
ivf_check_with_msg(ret, "Failed to load path in MemoryReadStorage for %s",
|
||||
IndexError::What(ret));
|
||||
|
||||
ailego::Params searcher_params;
|
||||
if (!searcher_class_.empty()) {
|
||||
searcher_params.set(PARAM_IVF_SEARCHER_OPTIMIZER, searcher_class_);
|
||||
}
|
||||
if (!searcher_params_.empty()) {
|
||||
searcher_params.set(PARAM_IVF_SEARCHER_OPTIMIZER_PARAMS, searcher_params_);
|
||||
}
|
||||
ret = this->load(container, searcher_params);
|
||||
ivf_check_with_msg(ret, "IVFCentroidIndex load failed with %s",
|
||||
IndexError::What(ret));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFCentroidIndex::load(const IndexStorage::Pointer &container,
|
||||
const ailego::Params params) {
|
||||
if (!container) {
|
||||
LOG_ERROR("Invalid container");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
int ret = IndexHelper::DeserializeFromStorage(container.get(), &meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to deserialize meta from container");
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto reformer_name = meta_.reformer_name();
|
||||
if (!reformer_name.empty()) {
|
||||
LOG_DEBUG("Load CentroidIndex with reformer %s, metric %s",
|
||||
reformer_name.c_str(), meta_.metric_name().c_str());
|
||||
if ((reformer_name == kInt8ReformerName ||
|
||||
reformer_name == kInt4ReformerName) &&
|
||||
meta_.metric_name() == kIPMetricName) {
|
||||
if (reformer_name == kInt8ReformerName) {
|
||||
reformer_ = std::make_shared<Int8QuantizerReformer4IP>();
|
||||
} else {
|
||||
reformer_ = std::make_shared<Int4QuantizerReformer4IP>();
|
||||
}
|
||||
if (!reformer_) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
} else {
|
||||
reformer_ = IndexFactory::CreateReformer(reformer_name);
|
||||
if (!reformer_) {
|
||||
LOG_ERROR("Failed to create reformer %s", reformer_name.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
}
|
||||
ret = reformer_->init(meta_.reformer_params());
|
||||
ivf_check_with_msg(ret, "Failed to initialize reformer %s",
|
||||
reformer_name.c_str());
|
||||
}
|
||||
|
||||
searcher_class_ = meta_.searcher_name();
|
||||
params.get(PARAM_IVF_SEARCHER_OPTIMIZER, &searcher_class_);
|
||||
params.get(PARAM_IVF_SEARCHER_OPTIMIZER_PARAMS, &searcher_params_);
|
||||
searcher_ = IndexFactory::CreateSearcher(searcher_class_);
|
||||
if (!searcher_) {
|
||||
LOG_ERROR("Failed to create searcher %s", searcher_class_.c_str());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
auto searcher_params = meta_.searcher_params();
|
||||
searcher_params.merge(searcher_params_);
|
||||
ret = searcher_->init(searcher_params);
|
||||
ivf_check_with_msg(ret, "Failed to initialize searcher %s",
|
||||
searcher_class_.c_str());
|
||||
|
||||
IndexMetric::Pointer metric;
|
||||
if (index_building_) {
|
||||
// The searcher index metric should specified in building process,
|
||||
// otherwise the query_metric will be used in searching
|
||||
metric = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
ivf_assert_with_msg(metric, IndexError_NoExist,
|
||||
"Failed to create metric %s",
|
||||
meta_.metric_name().c_str());
|
||||
ret = metric->init(meta_, meta_.metric_params());
|
||||
ivf_check_with_msg(ret, "Failed to initialize metric");
|
||||
}
|
||||
ret = searcher_->load(container, metric);
|
||||
ivf_check_with_msg(ret, "Failed to load searcher %s",
|
||||
searcher_class_.c_str());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
IndexTrainer::Pointer IVFCentroidIndex::prepare_trainer(
|
||||
const IndexCluster::CentroidList ¢roid_list) {
|
||||
IndexCluster::CentroidList level1_centroids;
|
||||
bool two_level = false;
|
||||
for (auto &it : centroid_list) {
|
||||
auto centroid = it;
|
||||
if (!centroid.subitems().empty()) {
|
||||
two_level = true;
|
||||
}
|
||||
centroid.mutable_subitems()->clear();
|
||||
centroid.mutable_similars()->clear();
|
||||
level1_centroids.emplace_back(centroid);
|
||||
}
|
||||
if (!two_level) {
|
||||
return IndexTrainer::Pointer();
|
||||
}
|
||||
|
||||
IndexBundle::Pointer bundle;
|
||||
IndexCluster::Serialize(meta_, level1_centroids, &bundle);
|
||||
return std::make_shared<FakeClusterTrainer>(meta_, bundle);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,247 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/container/params.h>
|
||||
#include "framework/index_framework.h"
|
||||
#include "ivf_params.h"
|
||||
#include "ivf_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! IVF Centroid Index
|
||||
*/
|
||||
class IVFCentroidIndex {
|
||||
public:
|
||||
typedef std::shared_ptr<IVFCentroidIndex> Pointer;
|
||||
|
||||
//! Constructor
|
||||
IVFCentroidIndex(void) {}
|
||||
|
||||
//! Destructor
|
||||
~IVFCentroidIndex(void) {
|
||||
IndexMemory *instance = IndexMemory::Instance();
|
||||
if (instance) {
|
||||
if (instance->has(path_)) {
|
||||
instance->remove(path_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//! Initialize
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms);
|
||||
|
||||
//! Set Quantizer for the index
|
||||
void set_quantizer(const std::string &quantizer_name,
|
||||
ailego::Params &quantizer_params) {
|
||||
quantizer_class_ = quantizer_name;
|
||||
quantizer_params_ = quantizer_params;
|
||||
}
|
||||
|
||||
//! Retrieve data address of the index
|
||||
const void *data(void) const {
|
||||
return data_;
|
||||
}
|
||||
|
||||
//! Retrieve size of the index
|
||||
size_t size(void) const {
|
||||
return size_;
|
||||
}
|
||||
|
||||
//! Create searcher context for centroid index
|
||||
IndexSearcher::Context::Pointer create_context() const {
|
||||
return searcher_ ? searcher_->create_context() : nullptr;
|
||||
}
|
||||
|
||||
//! Similarity search
|
||||
int search(const void *query, const IndexQueryMeta &qmeta, size_t count,
|
||||
IndexSearcher::Context::Pointer &ctx);
|
||||
|
||||
//! Search the nearest point, must be called in local thread pool
|
||||
uint32_t search_nearest_centroid(const void *query, size_t len);
|
||||
|
||||
//! Transform Data and Search the nearest point, called while adding record
|
||||
uint32_t transform_and_search_nearest_centroid(
|
||||
const void *record, const IndexQueryMeta &rmeta,
|
||||
IndexSearcher::Context::Pointer &ctx) const;
|
||||
|
||||
//! Build Centroid Index From Centroid List
|
||||
int build(const IndexCluster::CentroidList ¢roid_list);
|
||||
|
||||
//! Load Centroid Index From container
|
||||
int load(const IndexStorage::Pointer &container, const ailego::Params params);
|
||||
|
||||
//! Retrieve centroid count of the index
|
||||
size_t centroids_count(void) const {
|
||||
return centroids_count_;
|
||||
}
|
||||
|
||||
//! Retrieve meta
|
||||
const IndexMeta &meta() const {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
//! Retrieve reformer of the index
|
||||
const IndexReformer::Pointer reformer(void) const {
|
||||
return reformer_;
|
||||
}
|
||||
|
||||
static constexpr uint32_t kInvalidID = std::numeric_limits<uint32_t>::max();
|
||||
|
||||
private:
|
||||
/*! Centroids IndexHolder
|
||||
*/
|
||||
class CentroidsIndexHolder : public IndexHolder {
|
||||
public:
|
||||
class Iterator : public IndexHolder::Iterator {
|
||||
public:
|
||||
//! Index Holder Iterator Pointer
|
||||
typedef std::unique_ptr<Iterator> Pointer;
|
||||
|
||||
//! Constructor
|
||||
Iterator(std::vector<const void *> *features) : features_(features) {}
|
||||
|
||||
//! Destructor
|
||||
virtual ~Iterator(void) {}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
virtual const void *data(void) const override {
|
||||
return (*features_)[id_];
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
return id_ < features_->size();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
return id_;
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
++id_;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
std::vector<const void *> *features_{nullptr};
|
||||
uint32_t id_{0};
|
||||
};
|
||||
|
||||
//! Constructor
|
||||
CentroidsIndexHolder(const IndexMeta &meta,
|
||||
const IndexCluster::CentroidList ¢roid_list)
|
||||
: dimension_(meta.dimension()),
|
||||
element_size_(meta.element_size()),
|
||||
data_type_(meta.data_type()) {
|
||||
using CentroidList = IndexCluster::CentroidList;
|
||||
|
||||
std::function<void(const CentroidList &)> get_leaf_features =
|
||||
[&](const CentroidList ¢s) {
|
||||
if (cents.empty()) {
|
||||
return;
|
||||
}
|
||||
for (const auto &it : cents) {
|
||||
if (it.subitems().empty()) {
|
||||
features_.emplace_back(it.feature());
|
||||
} else {
|
||||
get_leaf_features(it.subitems());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
get_leaf_features(centroid_list);
|
||||
}
|
||||
|
||||
//! Retrieve count of elements in holder (-1 indicates unknown)
|
||||
virtual size_t count(void) const override {
|
||||
return features_.size();
|
||||
}
|
||||
|
||||
//! Retrieve dimension
|
||||
virtual size_t dimension(void) const override {
|
||||
return dimension_;
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
virtual IndexMeta::DataType data_type(void) const override {
|
||||
return data_type_;
|
||||
}
|
||||
|
||||
//! Retrieve element size in bytes
|
||||
virtual size_t element_size(void) const override {
|
||||
return element_size_;
|
||||
}
|
||||
|
||||
//! Retrieve if it can multi-pass
|
||||
virtual bool multipass(void) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
|
||||
return IndexHolder::Iterator::Pointer(
|
||||
new CentroidsIndexHolder::Iterator(&features_));
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
std::vector<const void *> features_{};
|
||||
size_t dimension_{0};
|
||||
size_t element_size_{0};
|
||||
IndexMeta::DataType data_type_{IndexMeta::DataType::DT_UNDEFINED};
|
||||
};
|
||||
|
||||
int build_index(const IndexCluster::CentroidList ¢roid_list,
|
||||
const IndexDumper::Pointer &dumper);
|
||||
|
||||
//! Prepare trainer for clustering index
|
||||
IndexTrainer::Pointer prepare_trainer(
|
||||
const IndexCluster::CentroidList ¢roid_list);
|
||||
|
||||
//! Quantize the centroid vectors in holder
|
||||
IndexHolder::Pointer quantize_holder(const IndexHolder::Pointer &holder);
|
||||
|
||||
|
||||
private:
|
||||
//! Constants
|
||||
constexpr static const char *kDefaultBuilder = "FlatBuilder";
|
||||
constexpr static const char *kTempralPathPrefix = "IVF";
|
||||
|
||||
//! Members
|
||||
IndexMeta meta_{};
|
||||
|
||||
IndexSearcher::Pointer searcher_{};
|
||||
IndexReformer::Pointer reformer_{};
|
||||
std::string builder_class_{kDefaultBuilder};
|
||||
std::string searcher_class_{};
|
||||
std::string quantizer_class_{};
|
||||
|
||||
std::string path_{};
|
||||
|
||||
ailego::Params builder_params_{};
|
||||
ailego::Params searcher_params_{};
|
||||
ailego::Params quantizer_params_{};
|
||||
|
||||
const void *data_{};
|
||||
size_t size_{};
|
||||
size_t centroids_count_{0};
|
||||
bool index_building_{false};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "ivf_distance_calculator.h"
|
||||
#include <iostream>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
IVFDistanceCalculator::IVFDistanceCalculator(const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &metric,
|
||||
uint32_t block_vec_cnt)
|
||||
: metric_ptr_(metric), block_vec_cnt_(block_vec_cnt) {
|
||||
row_distance_ = metric->distance();
|
||||
distanceXx1_ = metric->distance_matrix(block_vec_cnt, 1);
|
||||
distances_.resize(33);
|
||||
for (size_t b = 32; b != 0; b /= 2) {
|
||||
distances_[b] = metric->distance_matrix(block_vec_cnt, b);
|
||||
}
|
||||
element_size_ = meta.element_size();
|
||||
dimension_ = meta.dimension();
|
||||
if (meta.major_order() == IndexMeta::MajorOrder::MO_COLUMN) {
|
||||
column_major_order_ = true;
|
||||
} else {
|
||||
column_major_order_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
IVFDistanceCalculator::~IVFDistanceCalculator() {
|
||||
row_distance_ = nullptr;
|
||||
distanceXx1_ = nullptr;
|
||||
distances_.clear();
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,185 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/utility/time_helper.h>
|
||||
#include "framework/index_framework.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class IVFDistanceCalculator {
|
||||
public:
|
||||
typedef std::shared_ptr<IVFDistanceCalculator> Pointer;
|
||||
|
||||
//! Constructor
|
||||
IVFDistanceCalculator(const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &metric,
|
||||
uint32_t block_vec_cnt);
|
||||
|
||||
virtual ~IVFDistanceCalculator();
|
||||
|
||||
public:
|
||||
inline void query_centroids_distance(const void *query, size_t qnum,
|
||||
const void *feature, size_t fnum,
|
||||
float *distances);
|
||||
|
||||
inline void query_centroids_distance(const void *query, const void *feature,
|
||||
size_t fnum, float *distances);
|
||||
|
||||
inline void query_features_distance(const void *query, const void *feature,
|
||||
size_t fnum, float *distances);
|
||||
|
||||
inline void query_features_distance(const void *query, const void *feature,
|
||||
bool column_major, size_t fnum,
|
||||
float *distances);
|
||||
|
||||
protected:
|
||||
//! Row Major Distances -> Online
|
||||
inline void row_major_distance(const void *query, size_t qnum,
|
||||
const void *feature, size_t fnum, float *out);
|
||||
|
||||
inline void row_major_distance(const void *query, const void *feature,
|
||||
size_t fnum, float *out);
|
||||
|
||||
template <size_t Q>
|
||||
inline void batch_query_centroids_distance(const void *query,
|
||||
const void *feature, size_t fnum,
|
||||
float *distances);
|
||||
|
||||
protected:
|
||||
IndexMetric::Pointer metric_ptr_{};
|
||||
IndexMetric::MatrixDistance row_distance_{nullptr};
|
||||
IndexMetric::MatrixDistance distanceXx1_{nullptr};
|
||||
std::vector<IndexMetric::MatrixDistance> distances_{};
|
||||
|
||||
size_t element_size_{0};
|
||||
size_t dimension_{0};
|
||||
uint32_t block_vec_cnt_{0};
|
||||
bool column_major_order_{false};
|
||||
};
|
||||
|
||||
void IVFDistanceCalculator::query_centroids_distance(const void *query,
|
||||
size_t qnum,
|
||||
const void *feature,
|
||||
size_t fnum,
|
||||
float *distances) {
|
||||
if (column_major_order_) {
|
||||
switch (qnum) {
|
||||
case 1:
|
||||
batch_query_centroids_distance<1>(query, feature, fnum, distances);
|
||||
break;
|
||||
case 16:
|
||||
batch_query_centroids_distance<16>(query, feature, fnum, distances);
|
||||
break;
|
||||
case 8:
|
||||
batch_query_centroids_distance<8>(query, feature, fnum, distances);
|
||||
break;
|
||||
case 4:
|
||||
batch_query_centroids_distance<4>(query, feature, fnum, distances);
|
||||
break;
|
||||
case 2:
|
||||
batch_query_centroids_distance<2>(query, feature, fnum, distances);
|
||||
break;
|
||||
case 32:
|
||||
batch_query_centroids_distance<32>(query, feature, fnum, distances);
|
||||
break;
|
||||
default:
|
||||
LOG_ERROR("Unsupported query num %zu.", qnum);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
const uint8_t *cur_query = reinterpret_cast<const uint8_t *>(query);
|
||||
for (size_t q = 0; q < qnum; ++q) {
|
||||
this->row_major_distance(cur_query, feature, fnum, distances);
|
||||
cur_query += element_size_;
|
||||
distances += block_vec_cnt_;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void IVFDistanceCalculator::query_centroids_distance(const void *query,
|
||||
const void *feature,
|
||||
size_t fnum,
|
||||
float *distances) {
|
||||
this->query_features_distance(query, feature, fnum, distances);
|
||||
}
|
||||
|
||||
void IVFDistanceCalculator::query_features_distance(const void *query,
|
||||
const void *feature,
|
||||
size_t fnum,
|
||||
float *distances) {
|
||||
if (column_major_order_) {
|
||||
if (fnum == block_vec_cnt_) {
|
||||
distanceXx1_(feature, query, dimension_, distances);
|
||||
} else {
|
||||
this->row_major_distance(query, feature, fnum, distances);
|
||||
}
|
||||
} else {
|
||||
this->row_major_distance(query, feature, fnum, distances);
|
||||
}
|
||||
}
|
||||
|
||||
void IVFDistanceCalculator::query_features_distance(const void *query,
|
||||
const void *feature,
|
||||
bool column_major,
|
||||
size_t fnum,
|
||||
float *distances) {
|
||||
if (column_major) {
|
||||
ailego_assert_with(fnum == block_vec_cnt_, "Invalid Block");
|
||||
distanceXx1_(feature, query, dimension_, distances);
|
||||
} else {
|
||||
this->row_major_distance(query, feature, fnum, distances);
|
||||
}
|
||||
}
|
||||
|
||||
template <size_t Q>
|
||||
void IVFDistanceCalculator::batch_query_centroids_distance(const void *query,
|
||||
const void *feature,
|
||||
size_t fnum,
|
||||
float *distances) {
|
||||
if (fnum == block_vec_cnt_) {
|
||||
distances_[Q](feature, query, dimension_, distances);
|
||||
} else {
|
||||
row_major_distance(query, Q, feature, fnum, distances);
|
||||
}
|
||||
}
|
||||
|
||||
void IVFDistanceCalculator::row_major_distance(const void *query, size_t qnum,
|
||||
const void *feature, size_t fnum,
|
||||
float *out) {
|
||||
const uint8_t *cur_query = reinterpret_cast<const uint8_t *>(query);
|
||||
for (size_t q = 0; q < qnum; ++q) {
|
||||
const uint8_t *tmp_feature = reinterpret_cast<const uint8_t *>(feature);
|
||||
float *cur_out = out + q * fnum;
|
||||
for (size_t f = 0; f < fnum; ++f) {
|
||||
row_distance_(cur_query, tmp_feature, dimension_, cur_out + f);
|
||||
tmp_feature += element_size_;
|
||||
}
|
||||
cur_query += element_size_;
|
||||
}
|
||||
}
|
||||
|
||||
void IVFDistanceCalculator::row_major_distance(const void *query,
|
||||
const void *feature, size_t fnum,
|
||||
float *out) {
|
||||
const uint8_t *cur_feature = reinterpret_cast<const uint8_t *>(feature);
|
||||
for (size_t f = 0; f < fnum; ++f) {
|
||||
row_distance_(query, cur_feature, dimension_, out + f);
|
||||
cur_feature += element_size_;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,409 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "ivf_dumper.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int IVFDumper::dump_inverted_vector(uint32_t inverted_list_id, uint64_t key,
|
||||
const void *vec) {
|
||||
int ret = this->check_dump_inverted_list(inverted_list_id);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
++inverted_lists_meta_[cur_list_id_].vector_count;
|
||||
++header_.total_vector_count;
|
||||
block_.emplace(key, vec, IndexMeta::MajorOrder::MO_ROW);
|
||||
if (block_.full()) {
|
||||
ret = this->dump_block();
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFDumper::dump_inverted_block(uint32_t inverted_list_id,
|
||||
const uint64_t *keys, const void *vecs,
|
||||
uint32_t vector_count, bool column_major) {
|
||||
int ret = this->check_dump_inverted_list(inverted_list_id);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
if (block_.match_order(column_major ? IndexMeta::MajorOrder::MO_COLUMN
|
||||
: IndexMeta::MajorOrder::MO_ROW) &&
|
||||
vector_count == block_.capacity()) {
|
||||
// Dump the block directly
|
||||
size_t size = vector_count * meta_.element_size();
|
||||
size_t pd_size = ailego_align(size, 32) - size;
|
||||
if (dumper_->write(vecs, size) != size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper_->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
if (pd_size > 0) {
|
||||
std::string padding(pd_size, '\0');
|
||||
if (dumper_->write(padding.data(), pd_size) != pd_size) {
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
std::copy(keys, keys + vector_count, std::back_inserter(keys_));
|
||||
++inverted_lists_meta_[cur_list_id_].block_count;
|
||||
++header_.block_count;
|
||||
header_.inverted_body_size += size;
|
||||
} else {
|
||||
size_t step_size = meta_.element_size();
|
||||
if (column_major) {
|
||||
step_size = IndexMeta::AlignSizeof(meta_.data_type());
|
||||
}
|
||||
for (size_t i = 0; i < vector_count; ++i) {
|
||||
auto v = reinterpret_cast<const char *>(vecs) + i * step_size;
|
||||
block_.emplace(keys[i], v,
|
||||
column_major ? IndexMeta::MajorOrder::MO_COLUMN
|
||||
: IndexMeta::MajorOrder::MO_ROW);
|
||||
if (block_.full()) {
|
||||
ret = this->dump_block();
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inverted_lists_meta_[cur_list_id_].vector_count += vector_count;
|
||||
header_.total_vector_count += vector_count;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFDumper::dump_container_segment(const IndexStorage::Pointer &container,
|
||||
const std::string &segmemt_id) {
|
||||
auto seg = container->get(segmemt_id, 2);
|
||||
if (!seg) {
|
||||
LOG_ERROR("Failed to fetch segment %s from %s", segmemt_id.c_str(),
|
||||
container->name().c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
const size_t batch_size = 32 * 1024;
|
||||
const size_t total_size = seg->data_size() + seg->padding_size();
|
||||
size_t off = 0;
|
||||
while (off < total_size) {
|
||||
const void *data = nullptr;
|
||||
size_t rd_size = std::min(batch_size, total_size - off);
|
||||
if (seg->read(off, &data, rd_size) != rd_size) {
|
||||
LOG_ERROR("Failed to read data, off=%zu size=%zu", off, rd_size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
if (dumper_->write(data, rd_size) != rd_size) {
|
||||
LOG_ERROR("Failed to write data, size=%zu", rd_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
off += rd_size;
|
||||
}
|
||||
|
||||
int ret = dumper_->append(segmemt_id, seg->data_size(), seg->padding_size(),
|
||||
seg->data_crc());
|
||||
ivf_check_with_msg(ret, "Failed to append %s", segmemt_id.c_str());
|
||||
|
||||
dumped_size_ += total_size;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFDumper::dump_inverted_vector_finished(void) {
|
||||
//! Dump Inverted Index Segment
|
||||
if (!block_.empty()) {
|
||||
int ret = this->dump_block();
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
header_.block_size = block_.block_size();
|
||||
size_t segment_size = header_.inverted_body_size;
|
||||
int ret = dumper_->append(IVF_INVERTED_BODY_SEG_ID, segment_size, 0, 0);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to append to segment %s, ret=%d",
|
||||
IVF_INVERTED_BODY_SEG_ID.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
dumped_size_ += segment_size;
|
||||
|
||||
//! Dump Inverted Index Header Segment
|
||||
std::string str;
|
||||
meta_.serialize(&str);
|
||||
header_.header_size = sizeof(header_) + str.size();
|
||||
header_.index_meta_size = str.size();
|
||||
header_.inverted_list_count = inverted_lists_meta_.size();
|
||||
if (dumper_->write(&header_, sizeof(header_)) != sizeof(header_)) {
|
||||
LOG_ERROR("Failed to write data, size %zu", sizeof(header_));
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
if (dumper_->write(str.data(), str.size()) != str.size()) {
|
||||
LOG_ERROR("Failed to write data, size %zu", str.size());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
size_t padding_size = 0;
|
||||
ret = this->dump_padding(header_.header_size, &padding_size);
|
||||
ivf_check_error_code(ret);
|
||||
ret = dumper_->append(IVF_INVERTED_HEADER_SEG_ID, header_.header_size,
|
||||
padding_size, 0);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to append to segment %s, ret:%d",
|
||||
IVF_INVERTED_HEADER_SEG_ID.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
dumped_size_ += header_.header_size + padding_size;
|
||||
|
||||
LOG_DEBUG(
|
||||
"Dump header info: blocks=%u block_size=%u block_vec_count=%u "
|
||||
"inverted_list_count=%u total_vecs=%u inverted_size=%zu",
|
||||
header_.block_count, header_.block_size, header_.block_vector_count,
|
||||
header_.inverted_list_count, header_.total_vector_count,
|
||||
static_cast<size_t>(header_.inverted_body_size));
|
||||
|
||||
//! Dump Inverted Lists Meta Segment
|
||||
segment_size = inverted_lists_meta_.size() * sizeof(InvertedListMeta);
|
||||
ret = this->dump_segment(IVF_INVERTED_META_SEG_ID,
|
||||
inverted_lists_meta_.data(), segment_size);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
//! Dump Keys Segment
|
||||
ret = this->dump_segment(IVF_KEYS_SEG_ID, keys_.data(),
|
||||
keys_.size() * sizeof(keys_[0]));
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
//! Dump Mapping Segment
|
||||
auto mapping = std::make_shared<std::vector<uint32_t>>();
|
||||
IVFUtility::Sort(keys_.data(), mapping.get(), keys_.size());
|
||||
ret = this->dump_segment(IVF_MAPPING_SEG_ID, mapping->data(),
|
||||
mapping->size() * sizeof(uint32_t));
|
||||
ivf_check_error_code(ret);
|
||||
mapping.reset();
|
||||
|
||||
//! Dump the Offsets Segment
|
||||
return this->dump_offsets_segment();
|
||||
}
|
||||
|
||||
int IVFDumper::dump_centroid_index(const void *data, size_t size) {
|
||||
int ret = this->dump_segment(IVF_CENTROID_SEG_ID, data, size);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFDumper::dump_quantizer_params(
|
||||
const std::vector<IndexConverter::Pointer> &quantizers) {
|
||||
if (meta_.reformer_name() != kInt8ReformerName &&
|
||||
meta_.reformer_name() != kInt4ReformerName) {
|
||||
// IntegerQuantizer params is support only
|
||||
return 0;
|
||||
}
|
||||
if (quantizers.size() == 1) {
|
||||
//! Donot dump, using reformer params in IndexMeta
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (quantizers.size() != header_.inverted_list_count) {
|
||||
LOG_ERROR("Mismatch size, quantizers=%zu, inverted_list_count=%u",
|
||||
quantizers.size(), header_.inverted_list_count);
|
||||
return IndexError_Logic;
|
||||
}
|
||||
bool int8_quantizer = meta_.reformer_name() == kInt8ReformerName;
|
||||
std::vector<InvertedIntegerQuantizerParams> params;
|
||||
params.resize(header_.inverted_list_count);
|
||||
for (size_t i = 0; i < quantizers.size(); ++i) {
|
||||
auto &p = quantizers[i]->meta().reformer_params();
|
||||
auto &scale_key = int8_quantizer ? INT8_QUANTIZER_REFORMER_SCALE
|
||||
: INT4_QUANTIZER_REFORMER_SCALE;
|
||||
auto &bias_key = int8_quantizer ? INT8_QUANTIZER_REFORMER_BIAS
|
||||
: INT4_QUANTIZER_REFORMER_BIAS;
|
||||
if (inverted_lists_meta_[i].vector_count > 0 &&
|
||||
(!p.has(scale_key) || !p.has(bias_key))) {
|
||||
LOG_ERROR("Miss reformer params %s or %s", bias_key.c_str(),
|
||||
scale_key.c_str());
|
||||
return IndexError_Logic;
|
||||
}
|
||||
|
||||
params[i].bias = p.get_as_float(bias_key);
|
||||
params[i].scale = p.get_as_float(scale_key);
|
||||
}
|
||||
|
||||
return this->dump_segment(
|
||||
int8_quantizer ? IVF_INT8_QUANTIZED_PARAMS_SEG_ID
|
||||
: IVF_INT4_QUANTIZED_PARAMS_SEG_ID,
|
||||
params.data(), params.size() * sizeof(InvertedIntegerQuantizerParams));
|
||||
}
|
||||
|
||||
int IVFDumper::dump_original_vector(const void *data, size_t size) {
|
||||
if (dumped_feature_count_ >= header_.total_vector_count) {
|
||||
LOG_ERROR("Dump too much orignal features, expect=%u",
|
||||
header_.total_vector_count);
|
||||
return IndexError_Logic;
|
||||
}
|
||||
|
||||
if (dumper_->write(data, size) != size) {
|
||||
LOG_ERROR("Dumper write features failed");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
dumped_features_size_ += size;
|
||||
++dumped_feature_count_;
|
||||
if (dumped_feature_count_ == header_.total_vector_count) {
|
||||
//! Dump features finished, dump the meta
|
||||
size_t padding_size = 0;
|
||||
int ret = this->dump_padding(size, &padding_size);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
ret = dumper_->append(IVF_FEATURES_SEG_ID, dumped_features_size_,
|
||||
padding_size, 0);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dumper append segment %s failed, ret:%d",
|
||||
IVF_FEATURES_SEG_ID.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
dumped_size_ += dumped_features_size_;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFDumper::check_dump_inverted_list(uint32_t inverted_list_id) {
|
||||
if (inverted_list_id < cur_list_id_) {
|
||||
LOG_ERROR("Invalid backward vector dumping, want=%u cur=%u",
|
||||
inverted_list_id, cur_list_id_);
|
||||
return IndexError_Logic;
|
||||
}
|
||||
if (inverted_list_id >= inverted_lists_meta_.size()) {
|
||||
LOG_ERROR("Invalid inverted_list_id=%u, lists_size=%zu", inverted_list_id,
|
||||
inverted_lists_meta_.size());
|
||||
return IndexError_Logic;
|
||||
}
|
||||
if (inverted_list_id != cur_list_id_) {
|
||||
//! flush previous inverted_list block
|
||||
int ret = this->dump_block();
|
||||
ivf_check_error_code(ret);
|
||||
for (auto idx = cur_list_id_ + 1; idx <= inverted_list_id; ++idx) {
|
||||
inverted_lists_meta_[idx].offset = header_.inverted_body_size;
|
||||
inverted_lists_meta_[idx].id_offset = header_.total_vector_count;
|
||||
}
|
||||
cur_list_id_ = inverted_list_id;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFDumper::dump_offsets_segment(void) const {
|
||||
bool col_pri = meta_.major_order() == IndexMeta::MajorOrder::MO_COLUMN;
|
||||
size_t total_size = 0;
|
||||
for (size_t i = 0; i < inverted_lists_meta_.size(); ++i) {
|
||||
std::vector<InvertedVecLocation> offsets;
|
||||
const auto &m = inverted_lists_meta_[i];
|
||||
size_t vec_cnt = m.vector_count;
|
||||
size_t idx = 0;
|
||||
uint64_t off = m.offset;
|
||||
size_t align_idx = vec_cnt - vec_cnt % block_vector_count_;
|
||||
for (size_t j = 0; j < vec_cnt; ++j) {
|
||||
if (col_pri && j < align_idx) {
|
||||
offsets.emplace_back(off + idx * block_.align_size(), true);
|
||||
} else {
|
||||
offsets.emplace_back(off + idx * block_.element_size(), false);
|
||||
}
|
||||
++idx;
|
||||
if (idx == block_vector_count_) {
|
||||
off += header_.block_size;
|
||||
idx = 0;
|
||||
}
|
||||
}
|
||||
if (idx != 0) {
|
||||
off += (vec_cnt - align_idx) * meta_.element_size();
|
||||
}
|
||||
|
||||
size_t len = offsets.size() * sizeof(offsets[0]);
|
||||
size_t actual_len = dumper_->write(offsets.data(), len);
|
||||
if (actual_len != len) {
|
||||
LOG_ERROR("Write offsets failed expect %zu, actual: %zu.", len,
|
||||
actual_len);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
total_size += len;
|
||||
}
|
||||
|
||||
size_t padding_size = 0;
|
||||
int ret = this->dump_padding(total_size, &padding_size);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
ret = dumper_->append(IVF_OFFSETS_SEG_ID, total_size, padding_size, 0);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dumper append segment %s failed, ret:%d",
|
||||
IVF_OFFSETS_SEG_ID.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
dumped_size_ += total_size + padding_size;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFDumper::dump_segment(const std::string &segment_id, const void *data,
|
||||
size_t size) const {
|
||||
size_t len = dumper_->write(data, size);
|
||||
if (len != size) {
|
||||
LOG_ERROR("Dump segment %s data failed, expect=%zu, actual=%zu",
|
||||
segment_id.c_str(), size, len);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
size_t padding_size = 0;
|
||||
int ret = this->dump_padding(size, &padding_size);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
uint32_t crc = ailego::Crc32c::Hash(data, size);
|
||||
ret = dumper_->append(segment_id, size, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
dumped_size_ += size + padding_size;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFDumper::dump_padding(size_t data_size, size_t *padding_size) const {
|
||||
*padding_size = IVFUtility::AlignedSize(data_size) - data_size;
|
||||
if (*padding_size == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::string padding(*padding_size, '\0');
|
||||
if (dumper_->write(padding.data(), *padding_size) != *padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", *padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFDumper::dump_block(void) {
|
||||
if (block_.empty()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t size = ailego_align(block_.bytes(), 32);
|
||||
if (dumper_->write(block_.data(), size) != size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper_->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
auto &keys = block_.keys();
|
||||
std::copy(keys.begin(), keys.end(), std::back_inserter(keys_));
|
||||
++inverted_lists_meta_[cur_list_id_].block_count;
|
||||
++header_.block_count;
|
||||
header_.inverted_body_size += size;
|
||||
block_.clear();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,260 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "framework/index_framework.h"
|
||||
#include "metric/metric_params.h"
|
||||
#include "quantizer/quantizer_params.h"
|
||||
#include "ivf_index_format.h"
|
||||
#include "ivf_params.h"
|
||||
#include "ivf_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Quantized Clustering Dumper
|
||||
*/
|
||||
class IVFDumper {
|
||||
public:
|
||||
typedef std::shared_ptr<IVFDumper> Pointer;
|
||||
|
||||
//! Vectors block
|
||||
class Block {
|
||||
public:
|
||||
//! Initialize block
|
||||
void init(const IndexMeta &meta, uint32_t max_vec_count) {
|
||||
element_size_ = meta.element_size();
|
||||
auto bsize = IVFUtility::AlignedSize(max_vec_count, element_size_);
|
||||
data_.resize(bsize);
|
||||
count_ = 0u;
|
||||
major_order_ = meta.major_order();
|
||||
align_size_ = IndexMeta::AlignSizeof(meta.data_type());
|
||||
units_ = element_size_ / align_size_;
|
||||
max_vec_count_ = max_vec_count;
|
||||
keys_.reserve(max_vec_count_);
|
||||
}
|
||||
|
||||
//! Add a vector to the block in row major order
|
||||
//! If the block is full and the block order is column, make a
|
||||
//! transpose
|
||||
void emplace(uint64_t key, const void *vec, IndexMeta::MajorOrder order) {
|
||||
switch (align_size_) {
|
||||
case 2:
|
||||
do_emplace<uint16_t>(vec, order);
|
||||
break;
|
||||
case 4:
|
||||
do_emplace<uint32_t>(vec, order);
|
||||
break;
|
||||
case 8:
|
||||
do_emplace<uint64_t>(vec, order);
|
||||
break;
|
||||
default:
|
||||
ailego_check_with(false, "Unsupport Aligned Size");
|
||||
}
|
||||
keys_.emplace_back(key);
|
||||
}
|
||||
|
||||
bool full(void) const {
|
||||
return count_ == max_vec_count_;
|
||||
}
|
||||
|
||||
const void *data(void) const {
|
||||
return data_.data();
|
||||
}
|
||||
|
||||
void clear(void) {
|
||||
count_ = 0u;
|
||||
keys_.clear();
|
||||
}
|
||||
|
||||
bool empty(void) const {
|
||||
return count_ == 0u;
|
||||
}
|
||||
|
||||
size_t size(void) const {
|
||||
return count_;
|
||||
}
|
||||
|
||||
size_t capacity(void) const {
|
||||
return max_vec_count_;
|
||||
}
|
||||
|
||||
size_t align_size(void) const {
|
||||
return align_size_;
|
||||
}
|
||||
|
||||
size_t element_size(void) const {
|
||||
return element_size_;
|
||||
}
|
||||
|
||||
//! Retrieve block data size
|
||||
size_t bytes(void) const {
|
||||
return element_size_ * count_;
|
||||
}
|
||||
|
||||
//! Retrieve max block size in bytes
|
||||
size_t block_size(void) const {
|
||||
return data_.size();
|
||||
}
|
||||
|
||||
IndexMeta::MajorOrder major_order(void) const {
|
||||
return major_order_;
|
||||
}
|
||||
|
||||
const std::vector<uint64_t> &keys(void) const {
|
||||
return keys_;
|
||||
}
|
||||
|
||||
bool match_order(IndexMeta::MajorOrder column_major) const {
|
||||
return major_order_ == column_major;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Transpose the block vectors
|
||||
void transpose() {
|
||||
std::vector<uint8_t> buf(data_.size());
|
||||
IVFUtility::Transpose(align_size_, data_.data(), count_, units_,
|
||||
buf.data());
|
||||
data_.swap(buf);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void do_emplace(const void *vec, IndexMeta::MajorOrder order) {
|
||||
ailego_assert_with(count_ < max_vec_count_, "emplace a full block");
|
||||
|
||||
T *dst = reinterpret_cast<T *>(data_.data() + element_size_ * count_);
|
||||
const T *src = reinterpret_cast<const T *>(vec);
|
||||
size_t step = order == IndexMeta::MO_ROW ? 1 : max_vec_count_;
|
||||
for (auto i = 0u; i < units_; ++i) {
|
||||
*dst = *src;
|
||||
dst++;
|
||||
src += step;
|
||||
}
|
||||
|
||||
count_++;
|
||||
if (full() && major_order_ == IndexMeta::MO_COLUMN) {
|
||||
transpose();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
std::vector<uint8_t> data_{};
|
||||
std::vector<uint64_t> keys_{};
|
||||
uint32_t count_{0u};
|
||||
uint32_t units_{0u};
|
||||
uint32_t align_size_{0u};
|
||||
uint32_t element_size_{0u};
|
||||
uint32_t max_vec_count_{0u};
|
||||
IndexMeta::MajorOrder major_order_{};
|
||||
};
|
||||
|
||||
//! Constructor
|
||||
IVFDumper(const IndexMeta &meta, const IndexDumper::Pointer &dumper,
|
||||
size_t inverted_list_count, size_t block_vector_count)
|
||||
: meta_(meta),
|
||||
dumper_(dumper),
|
||||
block_vector_count_(block_vector_count),
|
||||
inverted_lists_meta_(inverted_list_count) {
|
||||
block_.init(meta, block_vector_count_);
|
||||
header_.block_vector_count = block_vector_count_;
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
IVFDumper(const IndexMeta &meta, const IndexDumper::Pointer &dumper,
|
||||
size_t inverted_list_count)
|
||||
: IVFDumper(meta, dumper, inverted_list_count, kDefaultBlockCount) {}
|
||||
|
||||
//! Destructor
|
||||
~IVFDumper() {
|
||||
// Check the dumper status
|
||||
if (dumped_feature_count_ > 0 &&
|
||||
dumped_feature_count_ != header_.total_vector_count) {
|
||||
LOG_ERROR("Dumped features=%u mismatch from invertedVecs=%u",
|
||||
dumped_feature_count_, header_.total_vector_count);
|
||||
ailego_assert_with(false, "invalid status");
|
||||
}
|
||||
}
|
||||
|
||||
//! Dump a vector in row major order
|
||||
int dump_inverted_vector(uint32_t inverted_list_id, uint64_t key,
|
||||
const void *vec);
|
||||
|
||||
int dump_inverted_block(uint32_t inverted_list_id, const uint64_t *keys,
|
||||
const void *vecs, uint32_t vector_count,
|
||||
bool column_major);
|
||||
|
||||
//! Finish dump the inverted vectors
|
||||
int dump_inverted_vector_finished(void);
|
||||
|
||||
//! Dump the centroids index
|
||||
int dump_centroid_index(const void *data, size_t size);
|
||||
|
||||
//! Dump params for each inverted list quantizer
|
||||
int dump_quantizer_params(
|
||||
const std::vector<IndexConverter::Pointer> &quantizers);
|
||||
|
||||
//! Dump the original vector, which doesnot been quantized
|
||||
int dump_original_vector(const void *data, size_t size);
|
||||
|
||||
//! Retrieve total dumped size
|
||||
size_t dumped_size(void) const {
|
||||
return dumped_size_;
|
||||
}
|
||||
|
||||
//! Retrieve total dumped vector count
|
||||
size_t dumped_count(void) const {
|
||||
return header_.total_vector_count;
|
||||
}
|
||||
|
||||
//! Dump the segment from container
|
||||
int dump_container_segment(const IndexStorage::Pointer &container,
|
||||
const std::string &segmemt_id);
|
||||
|
||||
private:
|
||||
int check_dump_inverted_list(uint32_t inverted_list_id);
|
||||
|
||||
//! Dump offsets segment
|
||||
int dump_offsets_segment(void) const;
|
||||
|
||||
//! Dump a segment
|
||||
int dump_segment(const std::string &segment_id, const void *data,
|
||||
size_t size) const;
|
||||
|
||||
//! Dump segment padding
|
||||
int dump_padding(size_t data_size, size_t *padding_size) const;
|
||||
|
||||
//! Dump a vector block
|
||||
int dump_block(void);
|
||||
|
||||
private:
|
||||
//! Constants
|
||||
static constexpr size_t kDefaultBlockCount = 32u;
|
||||
|
||||
//! Members
|
||||
Block block_{}; // vectors grouped in block
|
||||
const IndexMeta meta_{}; // IndexMeta of the inverted index
|
||||
const IndexDumper::Pointer dumper_{};
|
||||
size_t block_vector_count_{kDefaultBlockCount};
|
||||
std::vector<InvertedListMeta> inverted_lists_meta_{};
|
||||
std::vector<uint64_t> keys_{};
|
||||
InvertedIndexHeader header_{};
|
||||
uint32_t cur_list_id_{0};
|
||||
uint32_t dumped_feature_count_{0};
|
||||
size_t dumped_features_size_{0};
|
||||
mutable size_t dumped_size_{0};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,973 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "ivf_entity.h"
|
||||
#include <iostream>
|
||||
#include "ivf_utility.h"
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! Initialize
|
||||
int IVFEntity::IVFReformerWrapper::init(const IndexMeta &imeta) {
|
||||
auto &name = imeta.reformer_name();
|
||||
|
||||
if (name.empty()) {
|
||||
type_ = kReformerTpNone;
|
||||
return 0;
|
||||
}
|
||||
|
||||
auto reformer = IndexFactory::CreateReformer(name);
|
||||
if (!reformer) {
|
||||
LOG_ERROR("Failed to create reformer %s", name.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
int ret = reformer->init(imeta.reformer_params());
|
||||
ivf_check_with_msg(ret, "Failed to init reformer %s", name.c_str());
|
||||
|
||||
reformer_ = std::move(reformer);
|
||||
|
||||
if (name == kInt8ReformerName) {
|
||||
if (imeta.metric_name() == kIPMetricName) {
|
||||
type_ = kReformerTpInnerProductInt8;
|
||||
return 0;
|
||||
}
|
||||
auto &key = INT8_QUANTIZER_REFORMER_SCALE;
|
||||
if (!imeta.reformer_params().has(key)) {
|
||||
LOG_ERROR("Missing param %s in reformer %s", key.c_str(), name.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
};
|
||||
float scale = imeta.reformer_params().get_as_float(key);
|
||||
reciprocal_ = scale == 0.0 ? 1.0 : (1.0 / scale);
|
||||
type_ = kReformerTpInt8;
|
||||
} else if (name == kInt4ReformerName) {
|
||||
if (imeta.metric_name() == kIPMetricName) {
|
||||
type_ = kReformerTpInnerProductInt4;
|
||||
return 0;
|
||||
}
|
||||
auto &key = INT4_QUANTIZER_REFORMER_SCALE;
|
||||
if (!imeta.reformer_params().has(key)) {
|
||||
LOG_ERROR("Missing param %s in reformer %s", key.c_str(), name.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
};
|
||||
float scale = imeta.reformer_params().get_as_float(key);
|
||||
reciprocal_ = scale == 0.0 ? 1.0 : (1.0 / scale);
|
||||
type_ = kReformerTpInt4;
|
||||
} else {
|
||||
type_ = kReformerTpDefault;
|
||||
}
|
||||
|
||||
LOG_DEBUG("Init QcReformer with %s, type=%u", name.c_str(), type_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Update the params, Called by gpu searcher only
|
||||
int IVFEntity::IVFReformerWrapper::update(const IndexMeta &meta) {
|
||||
auto &name = meta.reformer_name();
|
||||
if (name == kInt4ReformerName && meta.metric_name() == kL2MetricName) {
|
||||
auto &key = INT4_QUANTIZER_REFORMER_SCALE;
|
||||
if (!meta.reformer_params().has(key)) {
|
||||
LOG_ERROR("Missing param %s in reformer %s", key.c_str(), name.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
};
|
||||
float scale = meta.reformer_params().get_as_float(key);
|
||||
reciprocal_ = scale == 0.0 ? 1.0 : (1.0 / scale / kNormalizeScaleFactor);
|
||||
type_ = kReformerTpInt8;
|
||||
|
||||
ailego::Params params;
|
||||
float int8_scale = scale * kNormalizeScaleFactor;
|
||||
params.set(INT8_QUANTIZER_REFORMER_SCALE, int8_scale);
|
||||
float bias =
|
||||
meta.reformer_params().get_as_float(INT4_QUANTIZER_REFORMER_BIAS);
|
||||
params.set(INT8_QUANTIZER_REFORMER_BIAS, bias);
|
||||
params.set(
|
||||
INT4_QUANTIZER_REFORMER_METRIC,
|
||||
meta.reformer_params().get_as_string(INT4_QUANTIZER_REFORMER_METRIC));
|
||||
|
||||
auto reformer = IndexFactory::CreateReformer(kInt8ReformerName);
|
||||
if (!reformer) {
|
||||
LOG_ERROR("Failed to create reformer %s", name.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
int ret = reformer->init(params);
|
||||
ivf_check_with_msg(ret, "Failed to init reformer %s", name.c_str());
|
||||
|
||||
reformer_ = reformer;
|
||||
|
||||
LOG_DEBUG("Init QcReformer with %s, type=%u", name.c_str(), type_);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Transform a query
|
||||
int IVFEntity::IVFReformerWrapper::transform(const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
const void **out,
|
||||
IndexQueryMeta *ometa) {
|
||||
int ret = 0;
|
||||
|
||||
switch (type_) {
|
||||
case kReformerTpNone:
|
||||
*out = query;
|
||||
*ometa = qmeta;
|
||||
break;
|
||||
|
||||
case kReformerTpInnerProductInt8:
|
||||
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
scales_.resize(1);
|
||||
buffer_.resize(IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT8,
|
||||
qmeta.dimension()));
|
||||
this->transform(0, static_cast<const float *>(query), qmeta.dimension(),
|
||||
reinterpret_cast<int8_t *>(&buffer_[0]));
|
||||
*ometa = qmeta;
|
||||
ometa->set_meta(IndexMeta::DataType::DT_INT8, qmeta.dimension());
|
||||
*out = buffer_.data();
|
||||
break;
|
||||
|
||||
case kReformerTpInnerProductInt4:
|
||||
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
scales_.resize(1);
|
||||
buffer_.resize(IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT4,
|
||||
qmeta.dimension()));
|
||||
this->transform(0, static_cast<const float *>(query), qmeta.dimension(),
|
||||
reinterpret_cast<uint8_t *>(&buffer_[0]));
|
||||
*ometa = qmeta;
|
||||
ometa->set_meta(IndexMeta::DataType::DT_INT4, qmeta.dimension());
|
||||
*out = buffer_.data();
|
||||
break;
|
||||
|
||||
case kReformerTpInt8:
|
||||
case kReformerTpInt4:
|
||||
/* FALLTHRU */
|
||||
case kReformerTpDefault:
|
||||
ret = reformer_->transform(query, qmeta, &buffer_, ometa);
|
||||
*out = buffer_.data();
|
||||
break;
|
||||
|
||||
default:
|
||||
ret = IndexError_Unsupported;
|
||||
break;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! Transform querys
|
||||
int IVFEntity::IVFReformerWrapper::transform(const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
uint32_t count, const void **out,
|
||||
IndexQueryMeta *ometa) {
|
||||
int ret = 0;
|
||||
|
||||
switch (type_) {
|
||||
case kReformerTpNone:
|
||||
*out = query;
|
||||
*ometa = qmeta;
|
||||
break;
|
||||
|
||||
case kReformerTpInnerProductInt8:
|
||||
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
scales_.resize(count);
|
||||
buffer_.resize(count *
|
||||
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT8,
|
||||
qmeta.dimension()));
|
||||
{
|
||||
const float *ivec = reinterpret_cast<const float *>(query);
|
||||
int8_t *ovec = reinterpret_cast<int8_t *>(&buffer_[0]);
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
this->transform(i, &ivec[i * qmeta.dimension()], qmeta.dimension(),
|
||||
&ovec[i * qmeta.dimension()]);
|
||||
}
|
||||
}
|
||||
*ometa = qmeta;
|
||||
ometa->set_meta(IndexMeta::DataType::DT_INT8, qmeta.dimension());
|
||||
*out = buffer_.data();
|
||||
break;
|
||||
|
||||
case kReformerTpInnerProductInt4:
|
||||
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
scales_.resize(count);
|
||||
buffer_.resize(count *
|
||||
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT4,
|
||||
qmeta.dimension()));
|
||||
{
|
||||
const float *ivec = reinterpret_cast<const float *>(query);
|
||||
uint8_t *ovec = reinterpret_cast<uint8_t *>(&buffer_[0]);
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
this->transform(i, &ivec[i * qmeta.dimension()], qmeta.dimension(),
|
||||
&ovec[i * qmeta.dimension() / 2]);
|
||||
}
|
||||
}
|
||||
*ometa = qmeta;
|
||||
ometa->set_meta(IndexMeta::DataType::DT_INT4, qmeta.dimension());
|
||||
*out = buffer_.data();
|
||||
break;
|
||||
|
||||
case kReformerTpInt8:
|
||||
case kReformerTpInt4:
|
||||
/* FALLTHRU */
|
||||
case kReformerTpDefault:
|
||||
ret = reformer_->transform(query, qmeta, count, &buffer_, ometa);
|
||||
*out = buffer_.data();
|
||||
break;
|
||||
|
||||
default:
|
||||
ret = IndexError_Unsupported;
|
||||
break;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! Transform querys
|
||||
int IVFEntity::IVFReformerWrapper::transform_gpu(const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
const void **out,
|
||||
IndexQueryMeta *ometa) {
|
||||
int ret = 0;
|
||||
|
||||
switch (type_) {
|
||||
case kReformerTpNone:
|
||||
case kReformerTpDefault:
|
||||
*out = query;
|
||||
*ometa = qmeta;
|
||||
break;
|
||||
|
||||
case kReformerTpInnerProductInt4:
|
||||
case kReformerTpInnerProductInt8:
|
||||
if (qmeta.data_type() != IndexMeta::DataType::DT_FP32) {
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
scales_.resize(count);
|
||||
buffer_.resize(count *
|
||||
IndexMeta::ElementSizeof(IndexMeta::DataType::DT_INT8,
|
||||
qmeta.dimension()));
|
||||
{
|
||||
const float *ivec = reinterpret_cast<const float *>(query);
|
||||
int8_t *ovec = reinterpret_cast<int8_t *>(&buffer_[0]);
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
this->transform(i, &ivec[i * qmeta.dimension()], qmeta.dimension(),
|
||||
&ovec[i * qmeta.dimension()]);
|
||||
}
|
||||
}
|
||||
*ometa = qmeta;
|
||||
ometa->set_meta(IndexMeta::DataType::DT_INT8, qmeta.dimension());
|
||||
*out = buffer_.data();
|
||||
break;
|
||||
|
||||
case kReformerTpInt8:
|
||||
case kReformerTpInt4:
|
||||
ret = reformer_->transform(query, qmeta, count, &buffer_, ometa);
|
||||
*out = buffer_.data();
|
||||
break;
|
||||
|
||||
default:
|
||||
ret = IndexError_Unsupported;
|
||||
break;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
//! Convert a record
|
||||
int IVFEntity::IVFReformerWrapper::convert(const void *record,
|
||||
const IndexQueryMeta &rmeta,
|
||||
const void **out,
|
||||
IndexQueryMeta *ometa) {
|
||||
if (type_ == kReformerTpNone) {
|
||||
*out = record;
|
||||
*ometa = rmeta;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ret = reformer_->convert(record, rmeta, &buffer_, ometa);
|
||||
*out = buffer_.data();
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! Convert records
|
||||
int IVFEntity::IVFReformerWrapper::convert(const void *records,
|
||||
const IndexQueryMeta &rmeta,
|
||||
uint32_t count, const void **out,
|
||||
IndexQueryMeta *ometa) {
|
||||
if (type_ == kReformerTpNone) {
|
||||
*out = records;
|
||||
*ometa = rmeta;
|
||||
return 0;
|
||||
}
|
||||
int ret = reformer_->convert(records, rmeta, count, &buffer_, ometa);
|
||||
*out = buffer_.data();
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! Normalize score
|
||||
void IVFEntity::IVFReformerWrapper::normalize(size_t qidx,
|
||||
IndexDocumentHeap *heap) const {
|
||||
switch (type_) {
|
||||
case kReformerTpNone:
|
||||
return;
|
||||
|
||||
case kReformerTpInnerProductInt8:
|
||||
case kReformerTpInnerProductInt4:
|
||||
ailego_assert_with(qidx < scales_.size(), "invalid index");
|
||||
{
|
||||
auto reciprocal = 1.0f / scales_[qidx];
|
||||
for (auto &it : *heap) {
|
||||
*it.mutable_score() *= reciprocal;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case kReformerTpInt8:
|
||||
case kReformerTpInt4:
|
||||
for (auto &it : *heap) {
|
||||
*it.mutable_score() *= reciprocal_;
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
// Not support
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
//! Normalize score
|
||||
void IVFEntity::IVFReformerWrapper::normalize(size_t qidx, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
IndexDocumentHeap *heap) const {
|
||||
switch (type_) {
|
||||
case kReformerTpNone:
|
||||
return;
|
||||
|
||||
case kReformerTpInnerProductInt8:
|
||||
case kReformerTpInnerProductInt4:
|
||||
ailego_assert_with(qidx < scales_.size(), "invalid index");
|
||||
{
|
||||
auto reciprocal = 1.0f / scales_[qidx];
|
||||
for (auto &it : *heap) {
|
||||
*it.mutable_score() *= reciprocal;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case kReformerTpInt8:
|
||||
case kReformerTpInt4:
|
||||
for (auto &it : *heap) {
|
||||
*it.mutable_score() *= reciprocal_;
|
||||
}
|
||||
break;
|
||||
|
||||
case kReformerTpDefault:
|
||||
reformer_->normalize(query, qmeta, *heap);
|
||||
break;
|
||||
|
||||
default:
|
||||
// Not support
|
||||
LOG_ERROR("Not a supported type in QC reformer, type: %u", type_);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void IVFEntity::IVFReformerWrapper::transform(size_t qidx, const float *in,
|
||||
size_t dim, int8_t *out) {
|
||||
ailego_assert_with(qidx < scales_.size(), "invalid index");
|
||||
|
||||
float abs_max = 0.0f;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
auto abs = std::abs(in[i]);
|
||||
if (abs > abs_max) {
|
||||
abs_max = abs;
|
||||
}
|
||||
}
|
||||
|
||||
if (abs_max > 0.0f) {
|
||||
float scale = 127 / abs_max;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = static_cast<int8_t>(std::round(in[i] * scale));
|
||||
}
|
||||
scales_[qidx] = scale;
|
||||
} else {
|
||||
std::fill(out, out + dim, static_cast<int8_t>(1));
|
||||
scales_[qidx] = std::numeric_limits<float>::max();
|
||||
}
|
||||
}
|
||||
|
||||
void IVFEntity::IVFReformerWrapper::transform(size_t qidx, const float *in,
|
||||
size_t dim, uint8_t *out) {
|
||||
ailego_assert_with(qidx < scales_.size(), "invalid index");
|
||||
ailego_assert_with(dim % 2 == 0, "invalid dim");
|
||||
|
||||
float abs_max = 0.0f;
|
||||
float max = -std::numeric_limits<float>::max();
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
float abs = std::abs(in[i]);
|
||||
abs_max = std::max(abs_max, abs);
|
||||
max = std::max(max, in[i]);
|
||||
}
|
||||
if (abs_max > 0.0f) {
|
||||
float scale = ((7 * abs_max > 8 * max) ? 8 : 7) / abs_max;
|
||||
for (size_t i = 0; i < dim; i += 2) {
|
||||
auto v1 = static_cast<int8_t>(std::round(in[i] * scale));
|
||||
auto v2 = static_cast<int8_t>(std::round(in[i + 1] * scale));
|
||||
out[i / 2] =
|
||||
(static_cast<uint8_t>(v1) & 0xF) | (static_cast<uint8_t>(v2) << 4);
|
||||
}
|
||||
scales_[qidx] = scale;
|
||||
} else {
|
||||
std::fill(out, out + dim / 2, static_cast<uint8_t>(9));
|
||||
scales_[qidx] = std::numeric_limits<float>::max();
|
||||
}
|
||||
}
|
||||
|
||||
int IVFEntity::load_header(const IndexStorage::Pointer &container) {
|
||||
//! Load the Header Segment
|
||||
auto header = container->get(IVF_INVERTED_HEADER_SEG_ID);
|
||||
if (!header) {
|
||||
LOG_ERROR("Failed to get segment %s", IVF_INVERTED_HEADER_SEG_ID.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (header->data_size() < sizeof(header_)) {
|
||||
LOG_ERROR("Invalid format for segment %s",
|
||||
IVF_INVERTED_HEADER_SEG_ID.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
const void *data = nullptr;
|
||||
if (header->read(0, &data, header->data_size()) != header->data_size()) {
|
||||
LOG_ERROR("Failed to read data, segment %s",
|
||||
IVF_INVERTED_HEADER_SEG_ID.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
std::memcpy(&header_, data, sizeof(header_));
|
||||
if (header_.header_size < sizeof(header_) + header_.index_meta_size ||
|
||||
header_.header_size > header->data_size()) {
|
||||
LOG_ERROR("Invalid header size %u", header_.header_size);
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
//! Load the index meta
|
||||
if (!meta_.deserialize(
|
||||
reinterpret_cast<const uint8_t *>(data) + sizeof(header_),
|
||||
header_.index_meta_size)) {
|
||||
LOG_ERROR("Failed to deserialize index meta");
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
int ret = reformer_.init(meta_);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
//! Create the distance calculator
|
||||
auto metric = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!metric) {
|
||||
LOG_ERROR("Failed to create metric %s", meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
ret = metric->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize metric %s", meta_.metric_name().c_str());
|
||||
return ret;
|
||||
}
|
||||
calculator_ = std::make_shared<IVFDistanceCalculator>(
|
||||
meta_, metric->query_metric() ? metric->query_metric() : metric,
|
||||
header_.block_vector_count);
|
||||
if (!calculator_) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFEntity::load(const IndexStorage::Pointer &container) {
|
||||
int ret = this->load_header(container);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
//! Load the remaining segments
|
||||
container_ = container;
|
||||
size_t expect_size = header_.inverted_body_size;
|
||||
inverted_ = load_segment(IVF_INVERTED_BODY_SEG_ID, expect_size);
|
||||
if (!inverted_) {
|
||||
LOG_ERROR("Failed to load segment, inverted_size=%zu block_count=%u",
|
||||
static_cast<size_t>(header_.inverted_body_size),
|
||||
header_.block_count);
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
expect_size = header_.inverted_list_count * sizeof(InvertedListMeta);
|
||||
inverted_meta_ = load_segment(IVF_INVERTED_META_SEG_ID, expect_size);
|
||||
if (!inverted_meta_) {
|
||||
LOG_ERROR("Failed to load segment, inverted_lists=%u",
|
||||
header_.inverted_list_count);
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
expect_size = header_.total_vector_count * sizeof(uint64_t);
|
||||
keys_ = load_segment(IVF_KEYS_SEG_ID, expect_size);
|
||||
if (!keys_) {
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
expect_size = header_.total_vector_count * sizeof(InvertedVecLocation);
|
||||
offsets_ = load_segment(IVF_OFFSETS_SEG_ID, expect_size);
|
||||
if (!offsets_) {
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
expect_size = header_.total_vector_count * sizeof(uint32_t);
|
||||
mapping_ = load_segment(IVF_MAPPING_SEG_ID, expect_size);
|
||||
if (!mapping_) {
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
norm_value_sqrt_ =
|
||||
meta_.metric_name() == "Euclidean" || meta_.metric_name() == "Manhattan";
|
||||
if (container_->get(IVF_INT8_QUANTIZED_PARAMS_SEG_ID) ||
|
||||
container->get(IVF_INT4_QUANTIZED_PARAMS_SEG_ID)) {
|
||||
expect_size =
|
||||
header_.inverted_list_count * sizeof(InvertedIntegerQuantizerParams);
|
||||
auto &seg_id = meta_.reformer_name() == kInt8ReformerName
|
||||
? IVF_INT8_QUANTIZED_PARAMS_SEG_ID
|
||||
: IVF_INT4_QUANTIZED_PARAMS_SEG_ID;
|
||||
integer_quantizer_params_ = load_segment(seg_id, expect_size);
|
||||
if (!integer_quantizer_params_) {
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
norm_value_ = 0.0f;
|
||||
} else if (meta_.reformer_name() == kInt8ReformerName ||
|
||||
meta_.reformer_name() == kInt4ReformerName) {
|
||||
auto &scale_key = meta_.reformer_name() == kInt8ReformerName
|
||||
? INT8_QUANTIZER_REFORMER_SCALE
|
||||
: INT4_QUANTIZER_REFORMER_SCALE;
|
||||
auto scale = meta_.reformer_params().get_as_float(scale_key);
|
||||
norm_value_ = this->convert_to_normalize_value(scale);
|
||||
} else {
|
||||
norm_value_ = 1.0f;
|
||||
}
|
||||
|
||||
if (container_->get(IVF_FEATURES_SEG_ID)) {
|
||||
features_ = load_segment(IVF_FEATURES_SEG_ID, 0);
|
||||
if (!features_) {
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (features_->data_size() % vector_count() != 0) {
|
||||
LOG_ERROR("Invalid featureSegment size=%zu, totalVecs=%zu",
|
||||
features_->data_size(), vector_count());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
}
|
||||
|
||||
LOG_DEBUG(
|
||||
"Load inverted index done, docs=%u invertedListCnt=%u "
|
||||
"elementSize=%u metric=%s reformer=%s",
|
||||
header_.total_vector_count, header_.inverted_list_count,
|
||||
meta_.element_size(), meta_.metric_name().c_str(),
|
||||
meta_.reformer_name().c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFEntity::search(size_t inverted_list_id, const void *query,
|
||||
const IndexFilter &filter, uint32_t *scan_count,
|
||||
IndexDocumentHeap *heap,
|
||||
IndexContext::Stats *context_stats) const {
|
||||
ailego_assert_with(inverted_list_id < header_.inverted_list_count,
|
||||
"invalid id");
|
||||
auto list_meta = this->inverted_list_meta(inverted_list_id);
|
||||
ivf_assert(list_meta, IndexError_ReadData);
|
||||
|
||||
const void *data = nullptr;
|
||||
const size_t block_vecs = header_.block_vector_count;
|
||||
float distances[block_vecs];
|
||||
const size_t batch_size = kBatchBlocks;
|
||||
const size_t block_size = header_.block_size;
|
||||
const auto norm_val = this->inverted_list_normalize_value(inverted_list_id);
|
||||
for (size_t i = 0; i < list_meta->block_count; i += batch_size) {
|
||||
//! Read vecs
|
||||
const size_t off = list_meta->offset + i * block_size;
|
||||
const size_t blocks = std::min(batch_size, list_meta->block_count - i);
|
||||
const size_t size =
|
||||
std::min(blocks * block_size,
|
||||
static_cast<size_t>(header_.inverted_body_size - off));
|
||||
if (inverted_->read(off, &data, size) != size) {
|
||||
LOG_ERROR("Failed to read block, off=%zu, size=%zu", off, size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
//! Read keys
|
||||
size_t items = std::min(blocks * block_vecs,
|
||||
list_meta->vector_count - (i * block_vecs));
|
||||
auto keys = get_keys(list_meta->id_offset + i * block_vecs, items);
|
||||
if (!keys) {
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
//! Compute distances for each block
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const size_t vecs_count =
|
||||
std::min(block_vecs, list_meta->vector_count - (i + b) * block_vecs);
|
||||
auto block_keys = keys + b * block_vecs;
|
||||
size_t keeps = 0;
|
||||
ailego_assert_with(block_vecs < sizeof(keeps) * 8, "bits overflow");
|
||||
for (size_t k = 0; k < vecs_count; ++k) {
|
||||
if (!filter(block_keys[k])) {
|
||||
keeps |= (1 << k);
|
||||
} else {
|
||||
++(*context_stats->mutable_filtered_count());
|
||||
}
|
||||
}
|
||||
if (keeps == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const void *block_data = static_cast<const char *>(data) + b * block_size;
|
||||
calculator_->query_features_distance(query, block_data, vecs_count,
|
||||
distances);
|
||||
|
||||
*(context_stats->mutable_dist_calced_count()) += vecs_count;
|
||||
|
||||
uint32_t id_off = list_meta->id_offset + (i + b) * block_vecs;
|
||||
for (size_t k = 0; k < vecs_count; ++k) {
|
||||
if (keeps & (1 << k)) {
|
||||
if (block_keys[k] != kInvalidKey) {
|
||||
heap->emplace(block_keys[k], distances[k] * norm_val, id_off + k);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
*scan_count = list_meta->vector_count;
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! search in inverted list without filter
|
||||
int IVFEntity::search(size_t inverted_list_id, const void *query,
|
||||
uint32_t *scan_count, IndexDocumentHeap *heap,
|
||||
IndexContext::Stats *context_stats) const {
|
||||
ailego_assert_with(inverted_list_id < header_.inverted_list_count,
|
||||
"invalid id");
|
||||
auto list_meta = inverted_list_meta(inverted_list_id);
|
||||
ivf_assert(list_meta, IndexError_ReadData);
|
||||
|
||||
const void *data = nullptr;
|
||||
const size_t block_vecs = header_.block_vector_count;
|
||||
float distances[block_vecs];
|
||||
const size_t batch_size = kBatchBlocks;
|
||||
const size_t block_size = header_.block_size;
|
||||
const auto norm_val = this->inverted_list_normalize_value(inverted_list_id);
|
||||
for (size_t i = 0; i < list_meta->block_count; i += batch_size) {
|
||||
//! Read vecs
|
||||
const size_t off = list_meta->offset + i * block_size;
|
||||
const size_t blocks = std::min(batch_size, list_meta->block_count - i);
|
||||
const size_t size =
|
||||
std::min(blocks * block_size,
|
||||
static_cast<size_t>(header_.inverted_body_size - off));
|
||||
if (inverted_->read(off, &data, size) != size) {
|
||||
LOG_ERROR("Failed to read block, off=%zu, size=%zu", off, size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
//! Read keys
|
||||
size_t items = std::min(blocks * block_vecs,
|
||||
list_meta->vector_count - (i * block_vecs));
|
||||
auto keys = get_keys(list_meta->id_offset + i * block_vecs, items);
|
||||
if (!keys) {
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
//! Compute distances for each block
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const size_t vecs_count =
|
||||
std::min(block_vecs, list_meta->vector_count - (i + b) * block_vecs);
|
||||
auto block_keys = keys + b * block_vecs;
|
||||
const void *block_data = static_cast<const char *>(data) + b * block_size;
|
||||
calculator_->query_features_distance(query, block_data, vecs_count,
|
||||
distances);
|
||||
for (size_t k = 0; k < vecs_count; ++k) {
|
||||
if (block_keys[k] != kInvalidKey) {
|
||||
uint32_t id = list_meta->id_offset + (i + b) * block_vecs + k;
|
||||
heap->emplace(block_keys[k], distances[k] * norm_val, id);
|
||||
}
|
||||
}
|
||||
*(context_stats->mutable_dist_calced_count()) += vecs_count;
|
||||
}
|
||||
}
|
||||
|
||||
*scan_count = list_meta->vector_count;
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! search all inverted list with filter
|
||||
int IVFEntity::search(const void *query, const IndexFilter &filter,
|
||||
IndexDocumentHeap *heap,
|
||||
IndexContext::Stats *context_stats) const {
|
||||
for (size_t i = 0; i < header_.inverted_list_count; ++i) {
|
||||
uint32_t scan_count;
|
||||
int ret = this->search(i, query, filter, &scan_count, heap, context_stats);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! search all inverted list without filter
|
||||
int IVFEntity::search(const void *query, IndexDocumentHeap *heap,
|
||||
IndexContext::Stats *context_stats) const {
|
||||
for (size_t i = 0; i < header_.inverted_list_count; ++i) {
|
||||
uint32_t scan_count;
|
||||
int ret = this->search(i, query, &scan_count, heap, context_stats);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const void *IVFEntity::get_vector(size_t id) const {
|
||||
if (features_) {
|
||||
const void *data = nullptr;
|
||||
size_t element_size = features_->data_size() / vector_count();
|
||||
size_t off = id * element_size;
|
||||
if (features_->read(off, &data, element_size) != element_size) {
|
||||
LOG_ERROR("Failed to read segment, off=%zu size=%zu", off, element_size);
|
||||
return nullptr;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
const void *data = nullptr;
|
||||
size_t size = sizeof(InvertedVecLocation);
|
||||
if (offsets_->read(id * size, &data, size) != size) {
|
||||
LOG_ERROR("Failed to read offsets segment, id=%zu", id);
|
||||
return nullptr;
|
||||
}
|
||||
auto &loc = *reinterpret_cast<const InvertedVecLocation *>(data);
|
||||
if (loc.column_major) {
|
||||
vector_.resize(meta_.element_size());
|
||||
auto unit_size = IndexMeta::AlignSizeof(meta_.data_type());
|
||||
size_t cols = meta_.element_size() / unit_size;
|
||||
size_t step = block_vector_count() * unit_size;
|
||||
size_t rd_size = step * (cols - 1) + unit_size;
|
||||
if (inverted_->read(loc.offset, &data, rd_size) != rd_size) {
|
||||
LOG_ERROR("Failed to read data, off=%zu size=%zu",
|
||||
static_cast<size_t>(loc.offset), rd_size);
|
||||
return nullptr;
|
||||
}
|
||||
for (size_t c = 0; c < cols; ++c) {
|
||||
vector_.replace(c * unit_size, unit_size,
|
||||
reinterpret_cast<const char *>(data) + c * step,
|
||||
unit_size);
|
||||
}
|
||||
return vector_.data();
|
||||
} else {
|
||||
if (inverted_->read(loc.offset, &data, meta_.element_size()) !=
|
||||
meta_.element_size()) {
|
||||
LOG_ERROR("Failed to read data, off=%zu size=%u",
|
||||
static_cast<size_t>(loc.offset), meta_.element_size());
|
||||
return nullptr;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
int IVFEntity::get_vector(size_t id, IndexStorage::MemoryBlock &block) const {
|
||||
if (features_) {
|
||||
size_t element_size = features_->data_size() / vector_count();
|
||||
size_t off = id * element_size;
|
||||
if (features_->read(off, block, element_size) != element_size) {
|
||||
LOG_ERROR("Failed to read segment, off=%zu size=%zu", off, element_size);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
IndexStorage::MemoryBlock data_block;
|
||||
size_t size = sizeof(InvertedVecLocation);
|
||||
if (offsets_->read(id * size, data_block, size) != size) {
|
||||
LOG_ERROR("Failed to read offsets segment, id=%zu", id);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
const void *data = data_block.data();
|
||||
auto &loc = *reinterpret_cast<const InvertedVecLocation *>(data);
|
||||
if (loc.column_major) {
|
||||
vector_.resize(meta_.element_size());
|
||||
auto unit_size = IndexMeta::AlignSizeof(meta_.data_type());
|
||||
size_t cols = meta_.element_size() / unit_size;
|
||||
size_t step = block_vector_count() * unit_size;
|
||||
size_t rd_size = step * (cols - 1) + unit_size;
|
||||
if (inverted_->read(loc.offset, &data, rd_size) != rd_size) {
|
||||
LOG_ERROR("Failed to read data, off=%zu size=%zu",
|
||||
static_cast<size_t>(loc.offset), rd_size);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
for (size_t c = 0; c < cols; ++c) {
|
||||
vector_.replace(c * unit_size, unit_size,
|
||||
reinterpret_cast<const char *>(data) + c * step,
|
||||
unit_size);
|
||||
}
|
||||
block.reset(vector_.data());
|
||||
return 0;
|
||||
} else {
|
||||
if (inverted_->read(loc.offset, block, meta_.element_size()) !=
|
||||
meta_.element_size()) {
|
||||
LOG_ERROR("Failed to read data, off=%zu size=%u",
|
||||
static_cast<size_t>(loc.offset), meta_.element_size());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t IVFEntity::key_to_id(uint64_t key) const {
|
||||
//! Do binary search
|
||||
uint32_t start = 0UL;
|
||||
uint32_t end = vector_count();
|
||||
const void *data = nullptr;
|
||||
uint32_t idx = 0u;
|
||||
while (start < end) {
|
||||
idx = start + (end - start) / 2;
|
||||
if (ailego_unlikely(mapping_->read(idx * sizeof(uint32_t), &data,
|
||||
sizeof(uint32_t)) != sizeof(uint32_t))) {
|
||||
LOG_ERROR("Failed to read mapping segment, idx=%u", idx);
|
||||
return std::numeric_limits<uint32_t>::max();
|
||||
}
|
||||
const uint64_t *mkey;
|
||||
uint32_t local_id = *reinterpret_cast<const uint32_t *>(data);
|
||||
if (ailego_unlikely(keys_->read(local_id * sizeof(uint64_t),
|
||||
(const void **)(&mkey),
|
||||
sizeof(uint64_t)) != sizeof(uint64_t))) {
|
||||
LOG_ERROR("Read key from segment failed");
|
||||
return std::numeric_limits<uint32_t>::max();
|
||||
}
|
||||
if (*mkey < key) {
|
||||
start = idx + 1;
|
||||
} else if (*mkey > key) {
|
||||
end = idx;
|
||||
} else {
|
||||
return local_id;
|
||||
}
|
||||
}
|
||||
return std::numeric_limits<uint32_t>::max();
|
||||
}
|
||||
|
||||
const void *IVFEntity::get_vector_by_key(uint64_t key) const {
|
||||
uint32_t id = this->key_to_id(key);
|
||||
if (id != std::numeric_limits<uint32_t>::max()) {
|
||||
return get_vector(id);
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
int IVFEntity::get_vector_by_key(uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const {
|
||||
uint32_t id = this->key_to_id(key);
|
||||
if (id != std::numeric_limits<uint32_t>::max()) {
|
||||
return get_vector(id, block);
|
||||
} else {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
|
||||
IVFEntity::Pointer IVFEntity::clone(void) const {
|
||||
auto entity = std::make_shared<IVFEntity>();
|
||||
return clone(entity);
|
||||
}
|
||||
|
||||
IVFEntity::Pointer IVFEntity::clone(const IVFEntity::Pointer &entity) const {
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to alloc IVFEntity");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto inverted = inverted_->clone();
|
||||
ivf_assert_with_msg(inverted, nullptr, "Failed to clone inverted segment");
|
||||
|
||||
auto inverted_meta = inverted_meta_->clone();
|
||||
ivf_assert_with_msg(inverted_meta, nullptr,
|
||||
"Failed to clone inverted meta segment");
|
||||
|
||||
auto keys = keys_->clone();
|
||||
ivf_assert_with_msg(keys, nullptr, "Failed to clone keys segment");
|
||||
|
||||
auto offsets = offsets_->clone();
|
||||
ivf_assert_with_msg(offsets, nullptr, "Failed to clone offsets segment");
|
||||
|
||||
auto mapping = mapping_->clone();
|
||||
ivf_assert_with_msg(mapping, nullptr, "Failed to clone mapping segment");
|
||||
|
||||
IndexStorage::Segment::Pointer integer_quantizer_params;
|
||||
if (integer_quantizer_params_) {
|
||||
integer_quantizer_params = integer_quantizer_params_->clone();
|
||||
if (!integer_quantizer_params) {
|
||||
LOG_ERROR("Failed to clone integer quantizer params segment");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
IndexStorage::Segment::Pointer features;
|
||||
if (features_) {
|
||||
features = features_->clone();
|
||||
if (!features) {
|
||||
LOG_ERROR("Failed to clone features segment");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
entity->meta_ = this->meta_;
|
||||
entity->reformer_ = this->reformer_;
|
||||
entity->calculator_ = this->calculator_;
|
||||
entity->header_ = this->header_;
|
||||
entity->container_ = this->container_;
|
||||
|
||||
entity->inverted_ = inverted;
|
||||
entity->inverted_meta_ = inverted_meta;
|
||||
entity->keys_ = keys;
|
||||
entity->offsets_ = offsets;
|
||||
entity->mapping_ = mapping;
|
||||
entity->integer_quantizer_params_ = integer_quantizer_params;
|
||||
entity->features_ = features;
|
||||
entity->norm_value_ = this->norm_value_;
|
||||
entity->norm_value_sqrt_ = this->norm_value_sqrt_;
|
||||
|
||||
return entity;
|
||||
}
|
||||
|
||||
IndexStorage::Segment::Pointer IVFEntity::load_segment(
|
||||
const std::string &seg_id, size_t expect_size) const {
|
||||
auto segment = container_->get(seg_id);
|
||||
if (!segment) {
|
||||
LOG_ERROR("Failed to get segment %s", seg_id.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
if (expect_size && segment->data_size() != expect_size) {
|
||||
LOG_ERROR("Invalid segment %s size=%zu, total_vecs=%u", seg_id.c_str(),
|
||||
segment->data_size(), header_.total_vector_count);
|
||||
return nullptr;
|
||||
}
|
||||
return segment;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,351 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "framework/index_framework.h"
|
||||
#include "metric/metric_params.h"
|
||||
#include "quantizer/quantizer_params.h"
|
||||
#include "ivf_distance_calculator.h"
|
||||
#include "ivf_index_format.h"
|
||||
#include "ivf_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! IVF Entity
|
||||
*/
|
||||
class IVFEntity {
|
||||
public:
|
||||
typedef std::shared_ptr<IVFEntity> Pointer;
|
||||
|
||||
class IVFReformerWrapper;
|
||||
|
||||
//! Constructor
|
||||
IVFEntity() {}
|
||||
|
||||
//! Destructor
|
||||
virtual ~IVFEntity() {}
|
||||
|
||||
//! Disable them
|
||||
IVFEntity(const IVFEntity &) = delete;
|
||||
IVFEntity &operator=(const IVFEntity &) = delete;
|
||||
|
||||
//! load the index from container
|
||||
virtual int load(const IndexStorage::Pointer &container);
|
||||
|
||||
//! search in inverted list with filter
|
||||
int search(size_t inverted_list_id, const void *query,
|
||||
const IndexFilter &filter, uint32_t *scan_count,
|
||||
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const;
|
||||
|
||||
//! search in inverted list without filter
|
||||
int search(size_t inverted_list_id, const void *query, uint32_t *scan_count,
|
||||
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const;
|
||||
|
||||
//! search all inverted list with filter
|
||||
int search(const void *query, const IndexFilter &filter,
|
||||
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const;
|
||||
|
||||
//! search all inverted list without filter
|
||||
int search(const void *query, IndexDocumentHeap *heap,
|
||||
IndexContext::Stats *context_stats) const;
|
||||
|
||||
//! Clone the entity
|
||||
virtual IVFEntity::Pointer clone(void) const;
|
||||
|
||||
//! Clone the entity
|
||||
IVFEntity::Pointer clone(const IVFEntity::Pointer &entity) const;
|
||||
|
||||
//! Retrieve the primary keys by local id in heap
|
||||
int retrieve_keys(IndexDocumentHeap *heap) const {
|
||||
for (auto &it : (*heap)) {
|
||||
uint64_t key = this->get_key(it.index());
|
||||
if (key == kInvalidKey) {
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
it.set_key(key);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve the total vectors in the index
|
||||
size_t vector_count(void) const {
|
||||
return header_.total_vector_count;
|
||||
}
|
||||
|
||||
//! Retrieve the inverted list count
|
||||
size_t inverted_list_count(void) const {
|
||||
return header_.inverted_list_count;
|
||||
}
|
||||
|
||||
//! Retrieve block size of the inverted vector
|
||||
size_t inverted_block_size(void) const {
|
||||
return header_.block_size;
|
||||
}
|
||||
|
||||
//! Retrieve the vectors count in one block
|
||||
size_t block_vector_count(void) const {
|
||||
return header_.block_vector_count;
|
||||
}
|
||||
|
||||
//! Retrieve IndexMeta of the inverted index
|
||||
const IndexMeta &meta(void) const {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
//! Retrieve a block of vectors
|
||||
const void *read_block(size_t inverted_list_id, size_t local_block_id,
|
||||
size_t *vecs_count) const {
|
||||
auto iv_meta = this->inverted_list_meta(inverted_list_id);
|
||||
if (!iv_meta || local_block_id >= iv_meta->block_count) {
|
||||
LOG_ERROR("Failed to read inverted list, listId=%zu blockIdx=%zu",
|
||||
inverted_list_id, local_block_id);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
size_t block_vecs = header_.block_vector_count;
|
||||
*vecs_count = std::min(block_vecs,
|
||||
iv_meta->vector_count - local_block_id * block_vecs);
|
||||
ailego_assert_with(*vecs_count <= header_.block_vector_count,
|
||||
"invalid vecs");
|
||||
const size_t off = iv_meta->offset + local_block_id * header_.block_size;
|
||||
const size_t size = *vecs_count * meta_.element_size();
|
||||
const void *data = nullptr;
|
||||
if (inverted_->read(off, &data, size) != size) {
|
||||
LOG_ERROR("Failed to read block off=%zu size=%zu", off, size);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
//! Retrieve the inverted list meta
|
||||
const InvertedListMeta *inverted_list_meta(size_t inverted_list_id) const {
|
||||
const void *data = nullptr;
|
||||
const size_t size = sizeof(InvertedListMeta);
|
||||
const size_t offset = inverted_list_id * size;
|
||||
if (inverted_meta_->read(offset, &data, size) != size) {
|
||||
LOG_ERROR("Failed to read inverted meta, id=%zu, size=%zu",
|
||||
inverted_list_id, size);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return static_cast<const InvertedListMeta *>(data);
|
||||
}
|
||||
|
||||
//! Retrieve the keys by consecutive local ids
|
||||
const uint64_t *get_keys(size_t id, size_t count) const {
|
||||
const void *data = nullptr;
|
||||
const size_t offset = id * sizeof(uint64_t);
|
||||
const size_t size = count * sizeof(uint64_t);
|
||||
if (keys_->read(offset, &data, size) != size) {
|
||||
LOG_ERROR("Failed to read keys, id=%zu, size=%zu", id, size);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return static_cast<const uint64_t *>(data);
|
||||
}
|
||||
|
||||
//! Retrieve the key by local id
|
||||
uint64_t get_key(size_t id) const {
|
||||
const void *data = nullptr;
|
||||
const size_t offset = id * sizeof(uint64_t);
|
||||
const size_t size = sizeof(uint64_t);
|
||||
if (keys_->read(offset, &data, size) != size) {
|
||||
LOG_ERROR("Failed to read key, id=%zu", id);
|
||||
return kInvalidKey;
|
||||
}
|
||||
|
||||
return *static_cast<const uint64_t *>(data);
|
||||
}
|
||||
|
||||
//! Retrieve vector by local id
|
||||
const void *get_vector(size_t id) const;
|
||||
|
||||
//! Retrieve vector by local id
|
||||
const void *get_vector_by_key(uint64_t key) const;
|
||||
|
||||
int get_vector(size_t id, IndexStorage::MemoryBlock &block) const;
|
||||
|
||||
int get_vector_by_key(uint64_t key, IndexStorage::MemoryBlock &block) const;
|
||||
|
||||
uint32_t key_to_id(uint64_t key) const;
|
||||
|
||||
//! Transform a query
|
||||
int transform(const void *query, const IndexQueryMeta &qmeta,
|
||||
const void **out, IndexQueryMeta *ometa) const {
|
||||
return reformer_.transform(query, qmeta, out, ometa);
|
||||
}
|
||||
|
||||
//! Transform queries
|
||||
int transform(const void *query, const IndexQueryMeta &qmeta, uint32_t count,
|
||||
const void **out, IndexQueryMeta *ometa) const {
|
||||
return reformer_.transform(query, qmeta, count, out, ometa);
|
||||
}
|
||||
|
||||
//! Normalize the score in query part
|
||||
void normalize(size_t qidx, IndexDocumentHeap *heap) const {
|
||||
return reformer_.normalize(qidx, heap);
|
||||
}
|
||||
|
||||
//! Retrieve the value for each inverted list to multiply for normalizing
|
||||
float inverted_list_normalize_value(size_t inverted_list_id) const {
|
||||
if (norm_value_ != 0.0f) {
|
||||
return norm_value_;
|
||||
}
|
||||
|
||||
// ailego_assert_with(integer_quantizer_params_, "nullptr");
|
||||
if (integer_quantizer_params_ != nullptr) {
|
||||
const void *data = nullptr;
|
||||
size_t size = sizeof(InvertedIntegerQuantizerParams);
|
||||
size_t off = inverted_list_id * size;
|
||||
if (integer_quantizer_params_->read(off, &data, size) != size) {
|
||||
LOG_ERROR("Failed to read data from segment, off=%zu", off);
|
||||
return 1.0f;
|
||||
}
|
||||
auto scale =
|
||||
static_cast<const InvertedIntegerQuantizerParams *>(data)->scale;
|
||||
return this->convert_to_normalize_value(scale);
|
||||
}
|
||||
|
||||
return norm_value_;
|
||||
}
|
||||
|
||||
//! Check whether the feature segment exist
|
||||
bool has_orignal_feature() const {
|
||||
return !!features_;
|
||||
}
|
||||
|
||||
//! Retrieve reformer
|
||||
const IVFReformerWrapper &reformer(void) const {
|
||||
return reformer_;
|
||||
}
|
||||
|
||||
/*! Index Reformer Wrapper
|
||||
* To transform query in inverted index searching, and normalize the score
|
||||
*/
|
||||
class IVFReformerWrapper {
|
||||
public:
|
||||
//! Constructor
|
||||
IVFReformerWrapper() {}
|
||||
|
||||
//! Assignment
|
||||
IVFReformerWrapper &operator=(const IVFReformerWrapper &wrapper) {
|
||||
reformer_ = wrapper.reformer_;
|
||||
type_ = wrapper.type_;
|
||||
buffer_.clear();
|
||||
buffer_.shrink_to_fit();
|
||||
reciprocal_ = wrapper.reciprocal_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Initialize
|
||||
int init(const IndexMeta &imeta);
|
||||
|
||||
//! Update
|
||||
int update(const IndexMeta &meta);
|
||||
|
||||
//! Transform a query
|
||||
int transform(const void *query, const IndexQueryMeta &qmeta,
|
||||
const void **out, IndexQueryMeta *ometa);
|
||||
|
||||
//! Transform queries
|
||||
int transform(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, const void **out, IndexQueryMeta *ometa);
|
||||
|
||||
//! Convert a record
|
||||
virtual int convert(const void *record, const IndexQueryMeta &rmeta,
|
||||
const void **out, IndexQueryMeta *ometa);
|
||||
|
||||
//! Convert records
|
||||
virtual int convert(const void *records, const IndexQueryMeta &rmeta,
|
||||
uint32_t count, const void **out,
|
||||
IndexQueryMeta *ometa);
|
||||
|
||||
//! Transform queries
|
||||
int transform_gpu(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, const void **out, IndexQueryMeta *ometa);
|
||||
|
||||
//! Normalize the score in query part
|
||||
void normalize(size_t qidx, IndexDocumentHeap *heap) const;
|
||||
|
||||
//! Normalize the score in query part
|
||||
void normalize(size_t qidx, const void *query, const IndexQueryMeta &qmeta,
|
||||
IndexDocumentHeap *heap) const;
|
||||
|
||||
private:
|
||||
//! Transform query from fp32 to int8
|
||||
void transform(size_t qidx, const float *in, size_t dim, int8_t *out);
|
||||
|
||||
//! Transform query from fp32 to int4
|
||||
void transform(size_t qidx, const float *in, size_t dim, uint8_t *out);
|
||||
|
||||
private:
|
||||
//! Constants
|
||||
enum Type {
|
||||
kReformerTpNone = 0,
|
||||
kReformerTpInnerProductInt8 = 1,
|
||||
kReformerTpInnerProductInt4 = 2,
|
||||
kReformerTpInt8 = 3,
|
||||
kReformerTpInt4 = 4,
|
||||
kReformerTpDefault = 7,
|
||||
};
|
||||
|
||||
//! Members
|
||||
Type type_{kReformerTpNone};
|
||||
IndexReformer::Pointer reformer_{};
|
||||
std::string buffer_{};
|
||||
float reciprocal_{0.0}; // for int8
|
||||
std::vector<float> scales_{}; // for int8 IP
|
||||
};
|
||||
|
||||
private:
|
||||
//! Load the segment by seg_id in expect_size segment size
|
||||
IndexStorage::Segment::Pointer load_segment(const std::string &seg_id,
|
||||
size_t expect_size) const;
|
||||
|
||||
//! Load the header segment
|
||||
int load_header(const IndexStorage::Pointer &container);
|
||||
|
||||
//! Convert the int8 quantizer scale to normalize value
|
||||
float convert_to_normalize_value(float scale) const {
|
||||
auto v = scale == 0.0 ? 1.0 : (1.0 / scale);
|
||||
return !norm_value_sqrt_ ? v : std::sqrt(v);
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Constants
|
||||
static constexpr size_t kBatchBlocks = 10u;
|
||||
|
||||
//! Members
|
||||
IndexMeta meta_{};
|
||||
mutable IVFReformerWrapper reformer_{};
|
||||
IVFDistanceCalculator::Pointer calculator_{};
|
||||
InvertedIndexHeader header_{};
|
||||
IndexStorage::Pointer container_{};
|
||||
IndexStorage::Segment::Pointer inverted_{};
|
||||
IndexStorage::Segment::Pointer inverted_meta_{};
|
||||
IndexStorage::Segment::Pointer keys_{};
|
||||
IndexStorage::Segment::Pointer offsets_{};
|
||||
IndexStorage::Segment::Pointer mapping_{};
|
||||
IndexStorage::Segment::Pointer features_{};
|
||||
IndexStorage::Segment::Pointer integer_quantizer_params_{};
|
||||
mutable std::string vector_{}; // temporary buffer for colomn major order
|
||||
float norm_value_{0.0f}; // normalize the inverted vector to orignal score
|
||||
bool norm_value_sqrt_{false}; // does the norm value need to sqrt
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/container/bitmap.h>
|
||||
#include "framework/index_framework.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
static constexpr uint64_t kInvalidKey = std::numeric_limits<uint64_t>::max();
|
||||
|
||||
/*! Index Format of Inverted Index Header
|
||||
*/
|
||||
struct InvertedIndexHeader {
|
||||
uint32_t header_size{0};
|
||||
uint32_t total_vector_count{0};
|
||||
uint64_t inverted_body_size{0};
|
||||
uint32_t inverted_list_count{0};
|
||||
uint32_t block_vector_count{0};
|
||||
uint32_t block_size{0};
|
||||
uint32_t block_count{0};
|
||||
uint32_t index_meta_size{0};
|
||||
char reserved_[28];
|
||||
char index_meta[0];
|
||||
};
|
||||
|
||||
/*! Index Format of Inverted Index Meta for each Inverted list
|
||||
*/
|
||||
struct InvertedListMeta {
|
||||
uint64_t offset{0};
|
||||
uint32_t block_count{0};
|
||||
uint32_t vector_count{0};
|
||||
uint32_t id_offset{0};
|
||||
char reserved_[16];
|
||||
};
|
||||
|
||||
/*! Index Format of Location in Inverted Index for each vector
|
||||
*/
|
||||
struct InvertedVecLocation {
|
||||
InvertedVecLocation(size_t off, bool col)
|
||||
: offset(off), column_major(col), reserved(0u) {}
|
||||
|
||||
uint64_t offset : 48; // feature offset in posting block segment
|
||||
uint64_t column_major : 1; // coloum major if true
|
||||
uint64_t reserved : 15;
|
||||
};
|
||||
|
||||
/*! Index Format of Integer Quantizer params for each inverted list
|
||||
*/
|
||||
struct InvertedIntegerQuantizerParams {
|
||||
float scale{1.0};
|
||||
float bias{0.0};
|
||||
};
|
||||
|
||||
/*! Location of Vectors Block in Storage Segment
|
||||
*/
|
||||
struct BlockLocation {
|
||||
uint16_t segment_id;
|
||||
uint16_t block_index;
|
||||
};
|
||||
|
||||
/*! The Header of a Block in Storage Segment
|
||||
*/
|
||||
struct BlockHeader {
|
||||
BlockLocation next;
|
||||
uint16_t vector_count;
|
||||
uint16_t column_major : 1;
|
||||
uint16_t reserved_ : 15;
|
||||
};
|
||||
|
||||
struct DeletionMap {
|
||||
void set(uint32_t index) {
|
||||
bitset.set(index);
|
||||
}
|
||||
|
||||
void reset(uint32_t index) {
|
||||
bitset.reset(index);
|
||||
}
|
||||
|
||||
bool test(uint32_t index) const {
|
||||
return bitset.test(index);
|
||||
}
|
||||
|
||||
bool is_dirty() const {
|
||||
return bitset.test_any();
|
||||
}
|
||||
|
||||
ailego::FixedBitset<32> bitset{};
|
||||
};
|
||||
|
||||
static_assert(sizeof(DeletionMap) == 4, "DeletionMap must be 4 bytes");
|
||||
|
||||
/*! Meta Information of Streamer Entity
|
||||
*/
|
||||
struct StreamerInvertedMeta {
|
||||
uint64_t create_time{0};
|
||||
uint64_t update_time{0};
|
||||
uint64_t revision_id{0};
|
||||
uint32_t segment_count{0};
|
||||
uint32_t segment_size{0};
|
||||
uint8_t reserved_[32];
|
||||
InvertedIndexHeader header;
|
||||
};
|
||||
|
||||
/*! Location of Vector in Storage Segment
|
||||
*/
|
||||
struct VectorLocation {
|
||||
//! Constructor
|
||||
VectorLocation(void) {}
|
||||
|
||||
//! Constructor
|
||||
VectorLocation(uint16_t id, bool col, uint32_t off)
|
||||
: segment_id(id), column_major(col), offset(off) {}
|
||||
|
||||
uint16_t segment_id;
|
||||
uint16_t column_major : 1;
|
||||
uint16_t reserved_ : 15;
|
||||
uint32_t offset;
|
||||
|
||||
public:
|
||||
bool operator==(const VectorLocation &other) const {
|
||||
return segment_id == other.segment_id &&
|
||||
column_major == other.column_major && offset == other.offset;
|
||||
}
|
||||
};
|
||||
|
||||
static_assert(sizeof(VectorLocation) == sizeof(uint64_t),
|
||||
"VectorLocation must be size of 8 bytes");
|
||||
|
||||
struct KeyInfo {
|
||||
KeyInfo(void) {}
|
||||
KeyInfo(uint32_t idx, const VectorLocation &loc)
|
||||
: centroid_idx(idx), location(loc) {}
|
||||
KeyInfo(VectorLocation loc) : location(loc) {}
|
||||
uint32_t centroid_idx;
|
||||
VectorLocation location;
|
||||
};
|
||||
|
||||
// Segments ID
|
||||
const std::string IVF_CENTROID_SEG_ID("ivf.centroid");
|
||||
const std::string IVF_INVERTED_BODY_SEG_ID("ivf.inverted_body");
|
||||
const std::string IVF_INVERTED_HEADER_SEG_ID("ivf.inverted_header");
|
||||
const std::string IVF_INVERTED_META_SEG_ID("ivf.inverted_meta");
|
||||
const std::string IVF_KEYS_SEG_ID("hc.keys");
|
||||
const std::string IVF_OFFSETS_SEG_ID("ivf.offsets");
|
||||
const std::string IVF_MAPPING_SEG_ID("ivf.mapping");
|
||||
const std::string IVF_FEATURES_SEG_ID("ivf.features");
|
||||
const std::string IVF_INT8_QUANTIZED_PARAMS_SEG_ID("ivf.int8_quantized_params");
|
||||
const std::string IVF_INT4_QUANTIZED_PARAMS_SEG_ID("ivf.int4_quantized_params");
|
||||
|
||||
const std::string IVF_INVERTED_LIST_HEAD_SEG_ID("ivf.inverted_list_head");
|
||||
const std::string IVF_STORAGE_SEGMENT_ID("ivf.S");
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,110 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "framework/index_searcher.h"
|
||||
#include "ivf_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! IVF IndexProvider
|
||||
*/
|
||||
class IVFIndexProvider : public IndexProvider {
|
||||
public:
|
||||
IVFIndexProvider(const IndexMeta &meta, const IVFEntity::Pointer &entity,
|
||||
const std::string &owner)
|
||||
: meta_(meta), entity_(entity), owner_class_(owner) {}
|
||||
|
||||
IVFIndexProvider(const IVFIndexProvider &) = delete;
|
||||
IVFIndexProvider &operator=(const IVFIndexProvider &) = delete;
|
||||
|
||||
public:
|
||||
//! Create a new iterator
|
||||
virtual Iterator::Pointer create_iterator(void) override {
|
||||
return Iterator::Pointer(new (std::nothrow) Iterator(entity_));
|
||||
}
|
||||
|
||||
//! Retrieve count of vectors
|
||||
virtual size_t count(void) const override {
|
||||
return entity_->vector_count();
|
||||
}
|
||||
|
||||
//! Retrieve dimension of vector
|
||||
virtual size_t dimension(void) const override {
|
||||
return meta_.dimension();
|
||||
}
|
||||
|
||||
//! Retrieve type of vector
|
||||
virtual IndexMeta::DataType data_type(void) const override {
|
||||
return meta_.data_type();
|
||||
}
|
||||
|
||||
//! Retrieve vector size in bytes
|
||||
virtual size_t element_size(void) const override {
|
||||
return meta_.element_size();
|
||||
}
|
||||
|
||||
//! Retrieve a vector using a primary key
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
return entity_->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
//! Retrieve the owner class
|
||||
virtual const std::string &owner_class(void) const override {
|
||||
return owner_class_;
|
||||
}
|
||||
|
||||
private:
|
||||
class Iterator : public IndexProvider::Iterator {
|
||||
public:
|
||||
Iterator(const IVFEntity::Pointer &entity) : entity_(entity) {}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
//! NOTICE: the vec feature will be changed after iterating to next, so
|
||||
//! the caller need to keep a copy of it before iterator to next vector
|
||||
virtual const void *data(void) const override {
|
||||
return entity_->get_vector(index_);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
return index_ < entity_->vector_count();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
return entity_->get_key(index_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
++index_;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
IVFEntity::Pointer entity_;
|
||||
size_t index_{0};
|
||||
};
|
||||
|
||||
private:
|
||||
//! Members
|
||||
const IndexMeta &meta_;
|
||||
IVFEntity::Pointer entity_;
|
||||
std::string owner_class_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,88 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
static const std::string SEPARATOR("/");
|
||||
static const std::string CENTROID_SEPERATOR = "*";
|
||||
|
||||
// builder params
|
||||
static const std::string PARAM_IVF_BUILDER_CENTROID_COUNT(
|
||||
"proxima.ivf.builder.centroid_count");
|
||||
static const std::string PARAM_IVF_BUILDER_CLUSTER_CLASS(
|
||||
"proxima.ivf.builder.cluster_class");
|
||||
static const std::string PARAM_IVF_BUILDER_THREAD_COUNT(
|
||||
"proxima.ivf.builder.thread_count");
|
||||
static const std::string PARAM_IVF_BUILDER_CLUSTER_AUTO_TUNING(
|
||||
"proxima.ivf.builder.cluster_auto_tuning");
|
||||
static const std::string PARAM_IVF_BUILDER_TRAIN_SAMPLE_COUNT(
|
||||
"proxima.ivf.builder.train_sample_count");
|
||||
static const std::string PARAM_IVF_BUILDER_TRAIN_SAMPLE_RATIO(
|
||||
"proxima.ivf.builder.train_sample_ratio");
|
||||
static const std::string PARAM_IVF_BUILDER_CONVERTER_PARAMS(
|
||||
"proxima.ivf.builder.converter_params");
|
||||
static const std::string PARAM_IVF_BUILDER_CONVERTER_CLASS(
|
||||
"proxima.ivf.builder.converter_class");
|
||||
static const std::string PARAM_IVF_BUILDER_STORE_ORIGINAL_FEATURES(
|
||||
"proxima.ivf.builder.store_original_features");
|
||||
static const std::string PARAM_IVF_BUILDER_QUANTIZER_CLASS(
|
||||
"proxima.ivf.builder.quantizer_class");
|
||||
static const std::string PARAM_IVF_BUILDER_QUANTIZE_BY_CENTROID(
|
||||
"proxima.ivf.builder.quantize_by_centroid");
|
||||
static const std::string PARAM_IVF_BUILDER_QUANTIZER_PARAMS(
|
||||
"proxima.ivf.builder.quantizer_params");
|
||||
static const std::string PARAM_IVF_BUILDER_CLUSTER_PARAMS_IN_LEVEL_PREFIX(
|
||||
"proxima.ivf.builder.cluster_params_in_level_");
|
||||
static const std::string PARAM_IVF_BUILDER_OPTIMIZER_CLASS(
|
||||
"proxima.ivf.builder.optimizer_class");
|
||||
static const std::string PARAM_IVF_BUILDER_OPTIMIZER_PARAMS(
|
||||
"proxima.ivf.builder.optimizer_params");
|
||||
static const std::string PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_CLASS(
|
||||
"proxima.ivf.builder.optimizer_quantizer_class");
|
||||
static const std::string PARAM_IVF_BUILDER_OPTIMIZER_QUANTIZER_PARAMS(
|
||||
"proxima.ivf.builder.optimizer_quantizer_params");
|
||||
static const std::string PARAM_IVF_BUILDER_BLOCK_VECTOR_COUNT(
|
||||
"proxima.ivf.builder.block_vector_count");
|
||||
|
||||
// searcher params
|
||||
static const std::string PARAM_IVF_SEARCHER_SCAN_RATIO(
|
||||
"proxima.ivf.searcher.scan_ratio");
|
||||
static const std::string PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD(
|
||||
"proxima.ivf.searcher.brute_force_threshold");
|
||||
static const std::string PARAM_IVF_SEARCHER_OPTIMIZER(
|
||||
"proxima.ivf.searcher.optimizer");
|
||||
static const std::string PARAM_IVF_SEARCHER_OPTIMIZER_PARAMS(
|
||||
"proxima.ivf.searcher.optimizer_params");
|
||||
static const std::string PARAM_IVF_SEARCHER_CONVERTER_REFORMER(
|
||||
"proxima.ivf.searcher.converter_reformer");
|
||||
|
||||
// Constants
|
||||
static constexpr char const *kIPMetricName = "InnerProduct";
|
||||
static constexpr char const *kMipsMetricName = "MipsSquaredEuclidean";
|
||||
static constexpr char const *kL2MetricName = "SquaredEuclidean";
|
||||
static constexpr char const *kMipsConverterName = "MipsConverter";
|
||||
static constexpr char const *kMipsRevConverterName = "MipsReverseConverter";
|
||||
static constexpr char const *kMipsReformerName = "MipsReformer";
|
||||
static constexpr char const *kInt8QuantizerName = "Int8QuantizerConverter";
|
||||
static constexpr char const *kInt4QuantizerName = "Int4QuantizerConverter";
|
||||
static constexpr char const *kInt8ReformerName = "Int8QuantizerReformer";
|
||||
static constexpr char const *kInt4ReformerName = "Int4QuantizerReformer";
|
||||
static constexpr float kNormalizeScaleFactor = 16.0f;
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,317 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "ivf_searcher.h"
|
||||
#include <ailego/utility/time_helper.h>
|
||||
#include <framework/index_segment_storage.h>
|
||||
#include "ivf_centroid_index.h"
|
||||
#include "ivf_index_provider.h"
|
||||
#include "ivf_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int IVFSearcher::init(const ailego::Params ¶meters) {
|
||||
params_ = parameters;
|
||||
|
||||
params_.get(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD, &bruteforce_threshold_);
|
||||
|
||||
searcher_state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFSearcher::cleanup(void) {
|
||||
this->unload();
|
||||
|
||||
params_.clear();
|
||||
bruteforce_threshold_ = kDefaultBfThreshold;
|
||||
|
||||
searcher_state_ = STATE_INIT;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFSearcher::load(IndexStorage::Pointer container,
|
||||
IndexMetric::Pointer /*metric*/) {
|
||||
if (!container) {
|
||||
LOG_ERROR("Invalid container");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (searcher_state_ != STATE_INITED) {
|
||||
LOG_ERROR("Initalize the searcher first before load index");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
int ret = IndexHelper::DeserializeFromStorage(container.get(), &meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to deserialize meta from container");
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! Load centroid index
|
||||
centroid_index_ = std::make_shared<IVFCentroidIndex>();
|
||||
if (!centroid_index_) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
auto seg = container->get(IVF_CENTROID_SEG_ID, 0);
|
||||
if (!seg) {
|
||||
LOG_ERROR("Failed to get segment %s", IVF_CENTROID_SEG_ID.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
IndexStorage::Pointer seg_container =
|
||||
std::make_shared<IndexSegmentStorage>(seg);
|
||||
if (!seg_container) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
ret = seg_container->open(std::string(), false);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexSegmentStorage load failed for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
ret = centroid_index_->load(seg_container, params_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to load index for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto reformer = centroid_index_->reformer();
|
||||
params_.set(PARAM_IVF_SEARCHER_CONVERTER_REFORMER, reformer);
|
||||
|
||||
//! load iverted index
|
||||
entity_ = std::make_shared<IVFEntity>();
|
||||
if (!entity_) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
ret = entity_->load(container);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
stats_.set_loaded_count(entity_->vector_count());
|
||||
stats_.set_loaded_costtime(timer.milli_seconds());
|
||||
|
||||
searcher_state_ = STATE_LOADED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFSearcher::unload(void) {
|
||||
magic_ = 0;
|
||||
centroid_index_.reset();
|
||||
entity_.reset();
|
||||
stats_.set_loaded_count(0UL);
|
||||
stats_.set_loaded_costtime(0UL);
|
||||
stats_.clear_attributes();
|
||||
searcher_state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFSearcher::search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
int IVFSearcher::search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (!query || qmeta.element_size() != meta_.element_size()) {
|
||||
LOG_ERROR("Null query or invalid qmeta");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
IVFSearcherContext *ctx = dynamic_cast<IVFSearcherContext *>(context.get());
|
||||
if (!ctx || ctx->topk() == 0) {
|
||||
LOG_ERROR("Invalid context or topk not set yet");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher
|
||||
int ret = this->update_context(ctx);
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
|
||||
ctx->reset_results(count);
|
||||
auto &entity = ctx->entity();
|
||||
auto &filter = ctx->filter();
|
||||
|
||||
//! Transform the querys for querying in inverted vector index later
|
||||
IndexQueryMeta iv_qmeta;
|
||||
int ret = entity->transform(query, qmeta, count, &query, &iv_qmeta);
|
||||
ivf_check_with_msg(ret, "Failed to transform querys");
|
||||
|
||||
// TODO: do batch search in matrix
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
auto &context_stats = ctx->mutable_stats(q);
|
||||
auto &heap = ctx->mutable_result_heap();
|
||||
heap.clear();
|
||||
if (!filter.is_valid()) {
|
||||
ret = entity->search(query, &heap, &context_stats);
|
||||
} else {
|
||||
ret = entity->search(query, filter, &heap, &context_stats);
|
||||
}
|
||||
ivf_check_with_msg(ret, "Failed to search in entity for %s",
|
||||
IndexError::What(ret));
|
||||
heap.sort(); // sort the results
|
||||
if (!filter.is_valid()) {
|
||||
// mapping the local id to key if query without filter
|
||||
ret = entity->retrieve_keys(&heap);
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
entity->normalize(q, &heap);
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + iv_qmeta.element_size();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFSearcher::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const {
|
||||
return this->search_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
int IVFSearcher::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const {
|
||||
if (entity_->vector_count() <= bruteforce_threshold_) {
|
||||
return this->search_bf_impl(query, qmeta, count, context);
|
||||
}
|
||||
if (!query || qmeta.element_size() != meta_.element_size()) {
|
||||
LOG_ERROR("Null query or invalid qmeta");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
IVFSearcherContext *ctx = dynamic_cast<IVFSearcherContext *>(context.get());
|
||||
if (!ctx || ctx->topk() == 0) {
|
||||
LOG_ERROR("Invalid context or topk not set yet");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher
|
||||
int ret = update_context(ctx);
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
|
||||
ctx->reset_results(count);
|
||||
auto &entity = ctx->entity();
|
||||
auto &filter = ctx->filter();
|
||||
|
||||
auto ¢roid_index_ctx = ctx->centroid_searcher_ctx();
|
||||
int ret = centroid_index_->search(query, qmeta, count, centroid_index_ctx);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
//! Transform the querys for querying in inverted vector index later
|
||||
IndexQueryMeta iv_qmeta;
|
||||
ret = entity->transform(query, qmeta, count, &query, &iv_qmeta);
|
||||
ivf_check_with_msg(ret, "Failed to transform querys");
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
auto ¢roids = centroid_index_ctx->result(q);
|
||||
auto &context_stats = ctx->mutable_stats(q);
|
||||
auto &heap = ctx->mutable_result_heap();
|
||||
heap.clear();
|
||||
uint32_t total_scan_count = 0;
|
||||
for (size_t i = 0;
|
||||
i < centroids.size() && total_scan_count < ctx->max_scan_count();
|
||||
++i) {
|
||||
auto cid = centroids[i].key();
|
||||
uint32_t scan_count = 0;
|
||||
if (!filter.is_valid()) {
|
||||
ret = entity->search(cid, query, &scan_count, &heap, &context_stats);
|
||||
} else {
|
||||
ret = entity->search(cid, query, filter, &scan_count, &heap,
|
||||
&context_stats);
|
||||
}
|
||||
ivf_check_with_msg(ret, "Failed to search in entity for %s",
|
||||
IndexError::What(ret));
|
||||
total_scan_count += scan_count;
|
||||
}
|
||||
heap.sort(); // sort the results
|
||||
if (!filter.is_valid()) {
|
||||
// mapping the local id to key if query without filter
|
||||
ret = entity->retrieve_keys(&heap);
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
entity->normalize(q, &heap);
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + iv_qmeta.element_size();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const IndexSearcher::Stats &IVFSearcher::stats(void) const {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
IndexSearcher::Context::Pointer IVFSearcher::create_context() const {
|
||||
if (searcher_state_ != STATE_LOADED) {
|
||||
LOG_ERROR("Load the index first before create context");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto entity = entity_->clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone IVFEntity");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto centroid_index_ctx = centroid_index_->create_context();
|
||||
if (!centroid_index_ctx) {
|
||||
LOG_ERROR("Failed to create centroid index context");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto context =
|
||||
new (std::nothrow) IVFSearcherContext(entity, centroid_index_ctx);
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to alloc IVFSearcherContext");
|
||||
return nullptr;
|
||||
}
|
||||
int ret = context->init(params_);
|
||||
if (ret != 0) {
|
||||
delete context;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
context->set_magic(magic_);
|
||||
|
||||
return Context::Pointer(context);
|
||||
}
|
||||
|
||||
IndexProvider::Pointer IVFSearcher::create_provider(void) const {
|
||||
if (searcher_state_ != STATE_LOADED) {
|
||||
LOG_ERROR("Load the index first before create provider");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto entity = entity_->clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone IVFEntity");
|
||||
return Provider::Pointer();
|
||||
}
|
||||
|
||||
auto *provider = new (std::nothrow)
|
||||
IVFIndexProvider(entity->has_orignal_feature() ? meta_ : entity->meta(),
|
||||
entity, "IVFSearcher");
|
||||
if (!provider) {
|
||||
LOG_ERROR("Failed to alloc IVFIndexProvider");
|
||||
return Provider::Pointer();
|
||||
}
|
||||
|
||||
return Provider::Pointer(provider);
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_SEARCHER(IVFSearcher);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,114 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "framework/index_searcher.h"
|
||||
#include "ivf_centroid_index.h"
|
||||
#include "ivf_entity.h"
|
||||
#include "ivf_searcher_context.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! IVF Searcher
|
||||
*/
|
||||
class IVFSearcher : public IndexSearcher {
|
||||
public:
|
||||
//! Initialize Searcher
|
||||
virtual int init(const ailego::Params ¶meters) override;
|
||||
|
||||
//! Cleanup Searcher
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Load index from container
|
||||
virtual int load(IndexStorage::Pointer container,
|
||||
IndexMetric::Pointer metric) override;
|
||||
|
||||
//! Unload index
|
||||
virtual int unload(void) override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override;
|
||||
|
||||
//! Create a searcher context
|
||||
virtual Context::Pointer create_context(void) const override;
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Retrieve meta of index
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
//! Retrieve params of index
|
||||
virtual const ailego::Params ¶ms(void) const override {
|
||||
return params_;
|
||||
}
|
||||
|
||||
protected:
|
||||
int update_context(IVFSearcherContext *ctx) const {
|
||||
auto entity = entity_->clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone QcEntity");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
//! The centroid index searcher may be different, so need to create one
|
||||
auto centroid_ctx = centroid_index_->create_context();
|
||||
if (!centroid_ctx) {
|
||||
LOG_ERROR("Failed to create centroid index searcher context");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return ctx->update_context(entity, centroid_ctx, params_, magic_);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Constants
|
||||
static constexpr uint32_t kDefaultBfThreshold = 1000u;
|
||||
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
|
||||
|
||||
//! Members
|
||||
IndexMeta meta_{};
|
||||
ailego::Params params_{};
|
||||
IVFCentroidIndex::Pointer centroid_index_{};
|
||||
IVFEntity::Pointer entity_{};
|
||||
uint32_t bruteforce_threshold_{kDefaultBfThreshold};
|
||||
uint32_t magic_{0};
|
||||
Stats stats_{};
|
||||
State searcher_state_{STATE_INIT};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,223 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/container/heap.h>
|
||||
#include "ivf_entity.h"
|
||||
#include "ivf_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! IVF Searcher Context
|
||||
*/
|
||||
class IVFSearcherContext : public IndexSearcher::Context {
|
||||
public:
|
||||
IVFSearcherContext(const IVFEntity::Pointer &ivf_entity,
|
||||
IndexSearcher::Context::Pointer ¢roid_ctx)
|
||||
: entity_(ivf_entity), centroid_searcher_ctx_(std::move(centroid_ctx)) {}
|
||||
|
||||
public:
|
||||
//! Set topk of search result
|
||||
virtual void set_topk(uint32_t k) override {
|
||||
topk_ = k;
|
||||
result_heap_.limit(topk_);
|
||||
result_heap_.set_threshold(this->threshold());
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search result with index
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
ailego_assert_with(results_.size() > idx, "invalid index");
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve mutable result with index
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
inline IndexDocumentHeap *result_heap() {
|
||||
return &result_heap_;
|
||||
}
|
||||
|
||||
//! Update the parameters of context
|
||||
virtual int update(const ailego::Params ¶ms) override {
|
||||
params.get(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD,
|
||||
&bruteforce_threshold_);
|
||||
params.get(PARAM_IVF_SEARCHER_SCAN_RATIO, &scan_ratio_);
|
||||
if (scan_ratio_ <= 0.0) {
|
||||
LOG_ERROR("Invalid params %s=%f", PARAM_IVF_SEARCHER_SCAN_RATIO.c_str(),
|
||||
scan_ratio_);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
size_t topk_val =
|
||||
std::max(static_cast<uint32_t>(
|
||||
std::round(entity_->inverted_list_count() * scan_ratio_)),
|
||||
1u);
|
||||
centroid_searcher_ctx_->set_topk(topk_val);
|
||||
max_scan_count_ =
|
||||
static_cast<uint32_t>(std::ceil(entity_->vector_count() * scan_ratio_));
|
||||
max_scan_count_ = std::max(bruteforce_threshold_, max_scan_count_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve magic number
|
||||
virtual uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
public:
|
||||
//! Initialize the context
|
||||
int init(const ailego::Params ¶ms) {
|
||||
return this->update(params);
|
||||
}
|
||||
|
||||
//! Update the magic number
|
||||
void set_magic(uint32_t mag) {
|
||||
magic_ = mag;
|
||||
}
|
||||
|
||||
//! Get Topk Value
|
||||
uint32_t topk() const override {
|
||||
return topk_;
|
||||
}
|
||||
|
||||
//! Retrieve scan ratio
|
||||
float scan_ratio(void) const {
|
||||
return scan_ratio_;
|
||||
}
|
||||
|
||||
//! Retrieve max scan count
|
||||
uint32_t max_scan_count(void) const {
|
||||
return max_scan_count_;
|
||||
}
|
||||
|
||||
uint32_t bruteforce_threshold() const {
|
||||
return bruteforce_threshold_;
|
||||
}
|
||||
|
||||
//! Retrieve magic number
|
||||
const IVFEntity::Pointer &entity() const {
|
||||
return entity_;
|
||||
}
|
||||
|
||||
//! Retrieve Mutable Query Result By Query Index
|
||||
IndexDocumentHeap &mutable_result_heap() {
|
||||
return result_heap_;
|
||||
}
|
||||
|
||||
void set_fetch_vector(bool v) override {
|
||||
fetch_vector_ = v;
|
||||
}
|
||||
|
||||
bool fetch_vector(void) const override {
|
||||
return fetch_vector_;
|
||||
}
|
||||
|
||||
//! Reset all the query results
|
||||
void reset_results(size_t qnum) {
|
||||
results_.resize(qnum);
|
||||
stats_vec_.resize(qnum);
|
||||
for (size_t i = 0; i < qnum; ++i) {
|
||||
results_[i].clear();
|
||||
stats_vec_[i].clear();
|
||||
}
|
||||
result_heap_.clear();
|
||||
result_heap_.limit(topk_);
|
||||
result_heap_.set_threshold(this->threshold());
|
||||
}
|
||||
|
||||
//! Update context, the context may be shared by different searcher
|
||||
int update_context(IVFEntity::Pointer &new_entity,
|
||||
IndexSearcher::Context::Pointer ¢roid_ctx,
|
||||
const ailego::Params ¶ms, uint32_t magic_num) {
|
||||
entity_ = new_entity;
|
||||
centroid_searcher_ctx_ = std::move(centroid_ctx);
|
||||
int ret = this->update(params);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
magic_ = magic_num;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve the centroid index context
|
||||
IndexSearcher::Context::Pointer ¢roid_searcher_ctx(void) {
|
||||
return centroid_searcher_ctx_;
|
||||
}
|
||||
|
||||
const Stats &stats(size_t idx = 0) const {
|
||||
ailego_assert_with(stats_vec_.size() > idx, "invalid index");
|
||||
return stats_vec_[idx];
|
||||
}
|
||||
|
||||
Stats &mutable_stats(size_t idx = 0) {
|
||||
ailego_assert_with(stats_vec_.size() > idx, "invalid index");
|
||||
return stats_vec_[idx];
|
||||
}
|
||||
|
||||
void topk_to_result(uint32_t idx) {
|
||||
if (ailego_unlikely(result_heap_.size() == 0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
int size = std::min(topk_, static_cast<uint32_t>(result_heap_.size()));
|
||||
result_heap_.sort();
|
||||
results_[idx].clear();
|
||||
for (int i = 0; i < size; ++i) {
|
||||
auto score = result_heap_[i].score();
|
||||
if (score > this->threshold()) {
|
||||
break;
|
||||
}
|
||||
|
||||
key_t key = result_heap_[i].key();
|
||||
if (fetch_vector_) {
|
||||
IndexStorage::MemoryBlock block;
|
||||
entity_->get_vector_by_key(key, block);
|
||||
results_[idx].emplace_back(key, score, key, block);
|
||||
} else {
|
||||
results_[idx].emplace_back(key, score);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
//! Constants
|
||||
static constexpr float kDefaultScanRatio = 0.1f;
|
||||
static constexpr uint32_t kDefaultBfThreshold = 1000u;
|
||||
|
||||
//! Members
|
||||
IVFEntity::Pointer entity_{};
|
||||
IndexSearcher::Context::Pointer centroid_searcher_ctx_{};
|
||||
IndexDocumentHeap result_heap_;
|
||||
std::vector<IndexDocumentList> results_{};
|
||||
std::vector<Stats> stats_vec_{};
|
||||
|
||||
bool fetch_vector_{false};
|
||||
uint32_t topk_{0};
|
||||
uint32_t magic_{0};
|
||||
float scan_ratio_{kDefaultScanRatio};
|
||||
uint32_t max_scan_count_{0};
|
||||
uint32_t bruteforce_threshold_{kDefaultBfThreshold};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,317 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "ivf_streamer.h"
|
||||
#include <ailego/utility/time_helper.h>
|
||||
#include <framework/index_segment_storage.h>
|
||||
#include "ivf_centroid_index.h"
|
||||
#include "ivf_index_provider.h"
|
||||
#include "ivf_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int IVFStreamer::init(const IndexMeta &meta, const ailego::Params ¶meters) {
|
||||
meta_ = meta;
|
||||
params_ = parameters;
|
||||
|
||||
params_.get(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD, &bruteforce_threshold_);
|
||||
|
||||
searcher_state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFStreamer::cleanup(void) {
|
||||
this->unload();
|
||||
|
||||
params_.clear();
|
||||
bruteforce_threshold_ = kDefaultBfThreshold;
|
||||
|
||||
searcher_state_ = STATE_INIT;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFStreamer::open(IndexStorage::Pointer storage) {
|
||||
if (!storage) {
|
||||
LOG_ERROR("Invalid storage");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (searcher_state_ != STATE_INITED) {
|
||||
LOG_ERROR("Initalize the searcher first before load index");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
int ret = IndexHelper::DeserializeFromStorage(storage.get(), &meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to deserialize meta from storage");
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! Load centroid index
|
||||
centroid_index_ = std::make_shared<IVFCentroidIndex>();
|
||||
if (!centroid_index_) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
auto seg = storage->get(IVF_CENTROID_SEG_ID, 0);
|
||||
if (!seg) {
|
||||
LOG_ERROR("Failed to get segment %s", IVF_CENTROID_SEG_ID.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
IndexStorage::Pointer seg_container =
|
||||
std::make_shared<IndexSegmentStorage>(seg);
|
||||
if (!seg_container) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
ret = seg_container->open(std::string(), false);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexSegmentStorage load failed for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
ret = centroid_index_->load(seg_container, params_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to load index for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto reformer = centroid_index_->reformer();
|
||||
params_.set(PARAM_IVF_SEARCHER_CONVERTER_REFORMER, reformer);
|
||||
|
||||
//! load iverted index
|
||||
entity_ = std::make_shared<IVFEntity>();
|
||||
if (!entity_) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
ret = entity_->load(storage);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
stats_.set_loaded_count(entity_->vector_count());
|
||||
stats_.set_loaded_costtime(timer.milli_seconds());
|
||||
|
||||
searcher_state_ = STATE_LOADED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFStreamer::unload(void) {
|
||||
magic_ = 0;
|
||||
centroid_index_.reset();
|
||||
entity_.reset();
|
||||
stats_.set_loaded_count(0UL);
|
||||
stats_.set_loaded_costtime(0UL);
|
||||
stats_.clear_attributes();
|
||||
searcher_state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFStreamer::search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
int IVFStreamer::search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (!query || qmeta.element_size() != meta_.element_size()) {
|
||||
LOG_ERROR("Null query or invalid qmeta");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
IVFSearcherContext *ctx = dynamic_cast<IVFSearcherContext *>(context.get());
|
||||
if (!ctx || ctx->topk() == 0) {
|
||||
LOG_ERROR("Invalid context or topk not set yet");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher
|
||||
int ret = this->update_context(ctx);
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
|
||||
ctx->reset_results(count);
|
||||
auto &entity = ctx->entity();
|
||||
auto &filter = ctx->filter();
|
||||
|
||||
//! Transform the querys for querying in inverted vector index later
|
||||
IndexQueryMeta iv_qmeta;
|
||||
int ret = entity->transform(query, qmeta, count, &query, &iv_qmeta);
|
||||
ivf_check_with_msg(ret, "Failed to transform querys");
|
||||
|
||||
// TODO: do batch search in matrix
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
auto &context_stats = ctx->mutable_stats(q);
|
||||
auto &heap = ctx->mutable_result_heap();
|
||||
heap.clear();
|
||||
if (!filter.is_valid()) {
|
||||
ret = entity->search(query, &heap, &context_stats);
|
||||
} else {
|
||||
ret = entity->search(query, filter, &heap, &context_stats);
|
||||
}
|
||||
ivf_check_with_msg(ret, "Failed to search in entity for %s",
|
||||
IndexError::What(ret));
|
||||
heap.sort(); // sort the results
|
||||
if (!filter.is_valid()) {
|
||||
// mapping the local id to key if query without filter
|
||||
ret = entity->retrieve_keys(&heap);
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
entity->normalize(q, &heap);
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + iv_qmeta.element_size();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IVFStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const {
|
||||
return this->search_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
int IVFStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const {
|
||||
if (entity_->vector_count() <= bruteforce_threshold_) {
|
||||
return this->search_bf_impl(query, qmeta, count, context);
|
||||
}
|
||||
if (!query || qmeta.element_size() != meta_.element_size()) {
|
||||
LOG_ERROR("Null query or invalid qmeta");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
IVFSearcherContext *ctx = dynamic_cast<IVFSearcherContext *>(context.get());
|
||||
if (!ctx || ctx->topk() == 0) {
|
||||
LOG_ERROR("Invalid context or topk not set yet");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher
|
||||
int ret = update_context(ctx);
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
|
||||
ctx->reset_results(count);
|
||||
auto &entity = ctx->entity();
|
||||
auto &filter = ctx->filter();
|
||||
|
||||
auto ¢roid_index_ctx = ctx->centroid_searcher_ctx();
|
||||
int ret = centroid_index_->search(query, qmeta, count, centroid_index_ctx);
|
||||
ivf_check_error_code(ret);
|
||||
|
||||
//! Transform the querys for querying in inverted vector index later
|
||||
IndexQueryMeta iv_qmeta;
|
||||
ret = entity->transform(query, qmeta, count, &query, &iv_qmeta);
|
||||
ivf_check_with_msg(ret, "Failed to transform querys");
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
auto ¢roids = centroid_index_ctx->result(q);
|
||||
auto &context_stats = ctx->mutable_stats(q);
|
||||
auto &heap = ctx->mutable_result_heap();
|
||||
heap.clear();
|
||||
uint32_t total_scan_count = 0;
|
||||
for (size_t i = 0;
|
||||
i < centroids.size() && total_scan_count < ctx->max_scan_count();
|
||||
++i) {
|
||||
auto cid = centroids[i].key();
|
||||
uint32_t scan_count = 0;
|
||||
if (!filter.is_valid()) {
|
||||
ret = entity->search(cid, query, &scan_count, &heap, &context_stats);
|
||||
} else {
|
||||
ret = entity->search(cid, query, filter, &scan_count, &heap,
|
||||
&context_stats);
|
||||
}
|
||||
ivf_check_with_msg(ret, "Failed to search in entity for %s",
|
||||
IndexError::What(ret));
|
||||
total_scan_count += scan_count;
|
||||
}
|
||||
heap.sort(); // sort the results
|
||||
if (!filter.is_valid()) {
|
||||
// mapping the local id to key if query without filter
|
||||
ret = entity->retrieve_keys(&heap);
|
||||
ivf_check_error_code(ret);
|
||||
}
|
||||
entity->normalize(q, &heap);
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + iv_qmeta.element_size();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const IndexSearcher::Stats &IVFStreamer::stats(void) const {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
IndexSearcher::Context::Pointer IVFStreamer::create_context() const {
|
||||
if (searcher_state_ != STATE_LOADED) {
|
||||
LOG_ERROR("Load the index first before create context");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto entity = entity_->clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone IVFEntity");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto centroid_index_ctx = centroid_index_->create_context();
|
||||
if (!centroid_index_ctx) {
|
||||
LOG_ERROR("Failed to create centroid index context");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto context =
|
||||
new (std::nothrow) IVFSearcherContext(entity, centroid_index_ctx);
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to alloc IVFSearcherContext");
|
||||
return nullptr;
|
||||
}
|
||||
int ret = context->init(params_);
|
||||
if (ret != 0) {
|
||||
delete context;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
context->set_magic(magic_);
|
||||
|
||||
return Context::Pointer(context);
|
||||
}
|
||||
|
||||
IndexProvider::Pointer IVFStreamer::create_provider(void) const {
|
||||
if (searcher_state_ != STATE_LOADED) {
|
||||
LOG_ERROR("Load the index first before create provider");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto entity = entity_->clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone IVFEntity");
|
||||
return Provider::Pointer();
|
||||
}
|
||||
|
||||
auto *provider = new (std::nothrow)
|
||||
IVFIndexProvider(entity->has_orignal_feature() ? meta_ : entity->meta(),
|
||||
entity, "IVFStreamer");
|
||||
if (!provider) {
|
||||
LOG_ERROR("Failed to alloc IVFIndexProvider");
|
||||
return Provider::Pointer();
|
||||
}
|
||||
|
||||
return Provider::Pointer(provider);
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_STREAMER(IVFStreamer);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#ifndef __IVF_STREAMER_H__
|
||||
#define __IVF_STREAMER_H__
|
||||
|
||||
#include "framework/index_streamer.h"
|
||||
#include "ivf_centroid_index.h"
|
||||
#include "ivf_entity.h"
|
||||
#include "ivf_searcher_context.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! IVF Searcher
|
||||
*/
|
||||
class IVFStreamer : public IndexStreamer {
|
||||
public:
|
||||
//! Initialize Searcher
|
||||
virtual int init(const IndexMeta & /*meta*/,
|
||||
const ailego::Params & /*params*/) override;
|
||||
|
||||
//! Cleanup Searcher
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Load index from container
|
||||
virtual int open(IndexStorage::Pointer storage) override;
|
||||
|
||||
virtual int flush(uint64_t /*check_point*/) override {
|
||||
return 0;
|
||||
}
|
||||
virtual int close(void) override {
|
||||
return this->unload();
|
||||
}
|
||||
|
||||
//! Unload index
|
||||
virtual int unload(void) override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override;
|
||||
|
||||
//! Create a searcher context
|
||||
virtual Context::Pointer create_context(void) const override;
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Retrieve meta of index
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
virtual int get_vector_by_id(
|
||||
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector_by_key(id, block);
|
||||
}
|
||||
|
||||
protected:
|
||||
int update_context(IVFSearcherContext *ctx) const {
|
||||
auto entity = entity_->clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone QcEntity");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
//! The centroid index searcher may be different, so need to create one
|
||||
auto centroid_ctx = centroid_index_->create_context();
|
||||
if (!centroid_ctx) {
|
||||
LOG_ERROR("Failed to create centroid index searcher context");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return ctx->update_context(entity, centroid_ctx, params_, magic_);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Constants
|
||||
static constexpr uint32_t kDefaultBfThreshold = 1000u;
|
||||
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
|
||||
|
||||
//! Members
|
||||
IndexMeta meta_{};
|
||||
ailego::Params params_{};
|
||||
IndexBuilder::Pointer builder_;
|
||||
IVFCentroidIndex::Pointer centroid_index_{};
|
||||
IVFEntity::Pointer entity_{};
|
||||
uint32_t bruteforce_threshold_{kDefaultBfThreshold};
|
||||
uint32_t magic_{0};
|
||||
Stats stats_{};
|
||||
State searcher_state_{STATE_INIT};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
|
||||
#endif //__IVF_STREAMER_H__
|
||||
@@ -0,0 +1,177 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <numeric>
|
||||
#include <vector>
|
||||
#include <ailego/utility/matrix_helper.h>
|
||||
#include <ailego/utility/time_helper.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
#ifndef ivf_check_error_code
|
||||
#define ivf_check_error_code(code) \
|
||||
if (ailego_unlikely((code) != 0)) return code
|
||||
#endif
|
||||
|
||||
#ifndef ivf_assert
|
||||
#define ivf_assert(cond, code) \
|
||||
if (ailego_unlikely(!(cond))) return code
|
||||
#endif
|
||||
|
||||
#ifndef ivf_check_with_msg
|
||||
#define ivf_check_with_msg(code, fmt, args...) \
|
||||
do { \
|
||||
if (ailego_unlikely((code) != 0)) { \
|
||||
LOG_ERROR(fmt, ##args); \
|
||||
return code; \
|
||||
} \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#ifndef ivf_assert_with_msg
|
||||
#define ivf_assert_with_msg(cond, err, fmt, args...) \
|
||||
do { \
|
||||
if (ailego_unlikely(!(cond))) { \
|
||||
LOG_ERROR(fmt, ##args); \
|
||||
return err; \
|
||||
} \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
/*! Quantized Clustering Utility
|
||||
*/
|
||||
class IVFUtility {
|
||||
public:
|
||||
//! Generator a random path with specificed prefix
|
||||
static inline std::string GenerateRandomPath(const std::string &prefix) {
|
||||
uint64_t timestamp = ailego::Monotime::MicroSeconds();
|
||||
return prefix + std::to_string(timestamp);
|
||||
}
|
||||
|
||||
//! Compute the default scan ratio for total vectors
|
||||
static inline float ComputeScanRatio(size_t vector_count) {
|
||||
// the fitting function for the follow points: 1000000(0.02)
|
||||
// 10000000(0.01) 50000000(0.005) 100000000(0.001)
|
||||
float scan_ratio = -0.004 * std::log(vector_count) + 0.0751;
|
||||
scan_ratio = std::max(scan_ratio, 0.0001f);
|
||||
return scan_ratio;
|
||||
}
|
||||
|
||||
//! Transpose the vectors in row major order to column major order
|
||||
static inline void Transpose(size_t align_size, const void *src, size_t m,
|
||||
size_t dim, void *dst);
|
||||
|
||||
//! Transpose the vectors in column major order to row major order
|
||||
static inline void ReverseTranspose(size_t align_size, const void *src,
|
||||
size_t m, size_t dim, void *dst);
|
||||
|
||||
//! Aligned size of a block vectors buffer
|
||||
static inline size_t AlignedSize(size_t fnum, size_t element_size);
|
||||
|
||||
//! Aligned size of one vector buffer
|
||||
static inline size_t AlignedSize(size_t element_size);
|
||||
|
||||
//! Sort arr with size in ascending order, and keep the index postion
|
||||
//! n2o keep the mapping: new position => origin postion
|
||||
//! For example, the input arr = [5, 3, 9, 6, 7], size = 5, after sort
|
||||
// arr = [3, 5, 6, 7, 9]
|
||||
// n2o = [1, 0, 3, 4, 2]
|
||||
//! To save memory, no extra memory is allocated
|
||||
template <typename T, typename I>
|
||||
static void Sort(T *arr, std::vector<I> *n2o, size_t size) {
|
||||
std::vector<I> o2n;
|
||||
o2n.resize(size);
|
||||
n2o->resize(size);
|
||||
|
||||
std::iota(n2o->begin(), n2o->end(), 0U);
|
||||
std::sort(n2o->begin(), n2o->end(),
|
||||
[&](I i, I j) { return arr[i] < arr[j]; });
|
||||
for (I i = 0U; i < size; ++i) {
|
||||
o2n[(*n2o)[i]] = i;
|
||||
}
|
||||
//! reorder arr in place, according to given n2o index
|
||||
for (I i = 0; i < size; ++i) {
|
||||
if (i != (*n2o)[i]) {
|
||||
T tmp = arr[i];
|
||||
I j = i, k;
|
||||
while (i != (k = (*n2o)[j])) {
|
||||
arr[j] = arr[k];
|
||||
(*n2o)[j] = j;
|
||||
j = k;
|
||||
}
|
||||
arr[j] = tmp;
|
||||
(*n2o)[j] = j;
|
||||
}
|
||||
}
|
||||
|
||||
for (I i = 0U; i < size; ++i) {
|
||||
(*n2o)[o2n[i]] = i;
|
||||
}
|
||||
}
|
||||
|
||||
//! Transpose one vector in block
|
||||
template <typename T>
|
||||
static inline void TransposeOne(const void *src, size_t M, size_t N,
|
||||
void *dst) {
|
||||
for (size_t i = 0; i < N; ++i) {
|
||||
reinterpret_cast<T *>(dst)[i] = reinterpret_cast<const T *>(src)[i * M];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void IVFUtility::Transpose(size_t align_size, const void *src, size_t m,
|
||||
size_t dim, void *dst) {
|
||||
switch (align_size) {
|
||||
case 2:
|
||||
ailego::MatrixHelper::Transpose<uint16_t>(src, m, dim, dst);
|
||||
break;
|
||||
case 4:
|
||||
ailego::MatrixHelper::Transpose<uint32_t>(src, m, dim, dst);
|
||||
break;
|
||||
case 8:
|
||||
ailego::MatrixHelper::Transpose<uint64_t>(src, m, dim, dst);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void IVFUtility::ReverseTranspose(size_t align_size, const void *src, size_t m,
|
||||
size_t dim, void *dst) {
|
||||
switch (align_size) {
|
||||
case 2:
|
||||
ailego::MatrixHelper::ReverseTranspose<uint16_t>(src, m, dim, dst);
|
||||
break;
|
||||
case 4:
|
||||
ailego::MatrixHelper::ReverseTranspose<uint32_t>(src, m, dim, dst);
|
||||
break;
|
||||
case 8:
|
||||
ailego::MatrixHelper::ReverseTranspose<uint64_t>(src, m, dim, dst);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
size_t IVFUtility::AlignedSize(size_t fnum, size_t element_size) {
|
||||
return ailego_align(fnum * element_size, 32);
|
||||
}
|
||||
|
||||
size_t IVFUtility::AlignedSize(size_t element_size) {
|
||||
return ailego_align(element_size, 32);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
Reference in New Issue
Block a user