feat: add diskann index (#369)
This commit is contained in:
@@ -8,6 +8,42 @@ cc_directory(ivf)
|
||||
cc_directory(hnsw)
|
||||
cc_directory(hnsw_sparse)
|
||||
cc_directory(vamana)
|
||||
|
||||
if(DISKANN_SUPPORTED)
|
||||
message(STATUS "build diskann")
|
||||
cc_directory(diskann)
|
||||
else()
|
||||
message(STATUS "not build diskann")
|
||||
# Empty stub library for unsupported platforms
|
||||
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/diskann_stub.cc
|
||||
"// Stub implementation for unsupported platforms\n"
|
||||
"// DiskAnn only supports Linux x86_64\n"
|
||||
"namespace zvec { namespace core { /* empty namespace for compatibility */ } }\n"
|
||||
)
|
||||
|
||||
if(MSVC)
|
||||
# MSVC: STATIC-only stub to avoid creating an empty DLL with no exports
|
||||
# (MSVC linker fails to produce an import library when there are zero exports)
|
||||
cc_library(
|
||||
NAME core_knn_diskann
|
||||
STATIC STRICT ALWAYS_LINK
|
||||
SRCS ${CMAKE_CURRENT_BINARY_DIR}/diskann_stub.cc
|
||||
LIBS core_framework
|
||||
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
else()
|
||||
cc_library(
|
||||
NAME core_knn_diskann
|
||||
STATIC SHARED STRICT ALWAYS_LINK
|
||||
SRCS ${CMAKE_CURRENT_BINARY_DIR}/diskann_stub.cc
|
||||
LIBS core_framework
|
||||
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(RABITQ_SUPPORTED)
|
||||
message(STATUS "BUILD RABITQ")
|
||||
cc_directory(hnsw_rabitq)
|
||||
|
||||
@@ -19,189 +19,195 @@ namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! General
|
||||
static const std::string GENERAL_CLUSTER_COUNT =
|
||||
"proxima.general.cluster.count";
|
||||
static const std::string GENERAL_CLUSTER_COUNT = "zvec.general.cluster.count";
|
||||
static const std::string GENERAL_THREAD_COUNT =
|
||||
"proxima.general.cluster.thread_count";
|
||||
"zvec.general.cluster.thread_count";
|
||||
|
||||
//! Optimize K-means
|
||||
static const std::string OPTKMEANS_CLUSTER_COUNT =
|
||||
"proxima.optkmeans.cluster.count";
|
||||
"zvec.optkmeans.cluster.count";
|
||||
static const std::string OPTKMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.optkmeans.cluster.max_iterations";
|
||||
"zvec.optkmeans.cluster.max_iterations";
|
||||
static const std::string OPTKMEANS_CLUSTER_EPSILON =
|
||||
"proxima.optkmeans.cluster.epsilon";
|
||||
"zvec.optkmeans.cluster.epsilon";
|
||||
static const std::string OPTKMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.optkmeans.cluster.shard_factor";
|
||||
"zvec.optkmeans.cluster.shard_factor";
|
||||
static const std::string OPTKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.optkmeans.cluster.purge_empty";
|
||||
"zvec.optkmeans.cluster.purge_empty";
|
||||
static const std::string OPTKMEANS_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.optkmeans.cluster.markov_chain_length";
|
||||
"zvec.optkmeans.cluster.markov_chain_length";
|
||||
static const std::string OPTKMEANS_CLUSTER_ASSUMPTION_FREE =
|
||||
"proxima.optkmeans.cluster.assumption_free";
|
||||
"zvec.optkmeans.cluster.assumption_free";
|
||||
|
||||
//! K-means
|
||||
static const std::string KMEANS_CLUSTER_COUNT = "proxima.kmeans.cluster.count";
|
||||
static const std::string KMEANS_CLUSTER_COUNT = "zvec.kmeans.cluster.count";
|
||||
static const std::string KMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmeans.cluster.shard_factor";
|
||||
static const std::string KMEANS_CLUSTER_EPSILON =
|
||||
"proxima.kmeans.cluster.epsilon";
|
||||
"zvec.kmeans.cluster.shard_factor";
|
||||
static const std::string KMEANS_CLUSTER_EPSILON = "zvec.kmeans.cluster.epsilon";
|
||||
static const std::string KMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.kmeans.cluster.max_iterations";
|
||||
"zvec.kmeans.cluster.max_iterations";
|
||||
static const std::string KMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.kmeans.cluster.purge_empty";
|
||||
static const std::string KMEANS_CLUSTER_BATCH = "proxima.kmeans.cluster.batch";
|
||||
"zvec.kmeans.cluster.purge_empty";
|
||||
static const std::string KMEANS_CLUSTER_BATCH = "zvec.kmeans.cluster.batch";
|
||||
static const std::string KMEANS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.kmeans.cluster.seeker_class";
|
||||
"zvec.kmeans.cluster.seeker_class";
|
||||
static const std::string KMEANS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.kmeans.cluster.seeker_params";
|
||||
"zvec.kmeans.cluster.seeker_params";
|
||||
|
||||
//! Mini Batch K-means
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.count";
|
||||
"zvec.minibatchkmeans.cluster.count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.minibatchkmeans.cluster.shard_factor";
|
||||
"zvec.minibatchkmeans.cluster.shard_factor";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_EPSILON =
|
||||
"proxima.minibatchkmeans.cluster.epsilon";
|
||||
"zvec.minibatchkmeans.cluster.epsilon";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.minibatchkmeans.cluster.max_iterations";
|
||||
"zvec.minibatchkmeans.cluster.max_iterations";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.minibatchkmeans.cluster.purge_empty";
|
||||
"zvec.minibatchkmeans.cluster.purge_empty";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_TRY_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.try_count";
|
||||
"zvec.minibatchkmeans.cluster.try_count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_BATCH_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.batch_count";
|
||||
"zvec.minibatchkmeans.cluster.batch_count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.minibatchkmeans.cluster.seeker_class";
|
||||
"zvec.minibatchkmeans.cluster.seeker_class";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.minibatchkmeans.cluster.seeker_params";
|
||||
"zvec.minibatchkmeans.cluster.seeker_params";
|
||||
|
||||
//! K-means++
|
||||
static const std::string KMEANSPP_CLUSTER_COUNT =
|
||||
"proxima.kmeanspp.cluster.count";
|
||||
static const std::string KMEANSPP_CLUSTER_COUNT = "zvec.kmeanspp.cluster.count";
|
||||
static const std::string KMEANSPP_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmeanspp.cluster.shard_factor";
|
||||
static const std::string KMEANSPP_CLUSTER_CLASS =
|
||||
"proxima.kmeanspp.cluster.class";
|
||||
"zvec.kmeanspp.cluster.shard_factor";
|
||||
static const std::string KMEANSPP_CLUSTER_CLASS = "zvec.kmeanspp.cluster.class";
|
||||
static const std::string KMEANSPP_CLUSTER_PARAMS =
|
||||
"proxima.kmeanspp.cluster.params";
|
||||
"zvec.kmeanspp.cluster.params";
|
||||
|
||||
//! K-MC2
|
||||
static const std::string KMC2_CLUSTER_COUNT = "proxima.kmc2.cluster.count";
|
||||
static const std::string KMC2_CLUSTER_COUNT = "zvec.kmc2.cluster.count";
|
||||
static const std::string KMC2_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmc2.cluster.shard_factor";
|
||||
"zvec.kmc2.cluster.shard_factor";
|
||||
static const std::string KMC2_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.kmc2.cluster.markov_chain_length";
|
||||
"zvec.kmc2.cluster.markov_chain_length";
|
||||
static const std::string KMC2_CLUSTER_ASSUMPTION_FREE =
|
||||
"proxima.kmc2.cluster.assumption_free";
|
||||
static const std::string KMC2_CLUSTER_CLASS = "proxima.kmc2.cluster.class";
|
||||
static const std::string KMC2_CLUSTER_PARAMS = "proxima.kmc2.cluster.params";
|
||||
"zvec.kmc2.cluster.assumption_free";
|
||||
static const std::string KMC2_CLUSTER_CLASS = "zvec.kmc2.cluster.class";
|
||||
static const std::string KMC2_CLUSTER_PARAMS = "zvec.kmc2.cluster.params";
|
||||
|
||||
//! Bisecting K-means
|
||||
static const std::string BIKMEANS_CLUSTER_COUNT =
|
||||
"proxima.bikmeans.cluster.count";
|
||||
static const std::string BIKMEANS_CLUSTER_COUNT = "zvec.bikmeans.cluster.count";
|
||||
static const std::string BIKMEANS_CLUSTER_INIT_COUNT =
|
||||
"proxima.bikmeans.cluster.init_count";
|
||||
"zvec.bikmeans.cluster.init_count";
|
||||
static const std::string BIKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.bikmeans.cluster.purge_empty";
|
||||
"zvec.bikmeans.cluster.purge_empty";
|
||||
static const std::string BIKMEANS_CLUSTER_FIRST_CLASS =
|
||||
"proxima.bikmeans.cluster.first_class";
|
||||
"zvec.bikmeans.cluster.first_class";
|
||||
static const std::string BIKMEANS_CLUSTER_SECOND_CLASS =
|
||||
"proxima.bikmeans.cluster.second_class";
|
||||
"zvec.bikmeans.cluster.second_class";
|
||||
static const std::string BIKMEANS_CLUSTER_FIRST_PARAMS =
|
||||
"proxima.bikmeans.cluster.first_params";
|
||||
"zvec.bikmeans.cluster.first_params";
|
||||
static const std::string BIKMEANS_CLUSTER_SECOND_PARAMS =
|
||||
"proxima.bikmeans.cluster.second_params";
|
||||
"zvec.bikmeans.cluster.second_params";
|
||||
|
||||
//! K-medoids
|
||||
static const std::string KMEDOIDS_CLUSTER_COUNT =
|
||||
"proxima.kmedoids.cluster.count";
|
||||
static const std::string KMEDOIDS_CLUSTER_COUNT = "zvec.kmedoids.cluster.count";
|
||||
static const std::string KMEDOIDS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmedoids.cluster.shard_factor";
|
||||
"zvec.kmedoids.cluster.shard_factor";
|
||||
static const std::string KMEDOIDS_CLUSTER_EPSILON =
|
||||
"proxima.kmedoids.cluster.epsilon";
|
||||
"zvec.kmedoids.cluster.epsilon";
|
||||
static const std::string KMEDOIDS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.kmedoids.cluster.max_iterations";
|
||||
"zvec.kmedoids.cluster.max_iterations";
|
||||
static const std::string KMEDOIDS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.kmedoids.cluster.purge_empty";
|
||||
"zvec.kmedoids.cluster.purge_empty";
|
||||
static const std::string KMEDOIDS_CLUSTER_BENCH_RATIO =
|
||||
"proxima.kmedoids.cluster.bench_ratio";
|
||||
"zvec.kmedoids.cluster.bench_ratio";
|
||||
static const std::string KMEDOIDS_CLUSTER_ONLY_MEANS =
|
||||
"proxima.kmedoids.cluster.only_means";
|
||||
"zvec.kmedoids.cluster.only_means";
|
||||
static const std::string KMEDOIDS_CLUSTER_WITHOUT_MEANS =
|
||||
"proxima.kmedoids.cluster.without_means";
|
||||
"zvec.kmedoids.cluster.without_means";
|
||||
static const std::string KMEDOIDS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.kmedoids.cluster.seeker_class";
|
||||
"zvec.kmedoids.cluster.seeker_class";
|
||||
static const std::string KMEDOIDS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.kmedoids.cluster.seeker_params";
|
||||
"zvec.kmedoids.cluster.seeker_params";
|
||||
|
||||
//! Stratified
|
||||
static const std::string STRATIFIED_CLUSTER_COUNT =
|
||||
"proxima.stratified.cluster.count";
|
||||
"zvec.stratified.cluster.count";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_CLASS =
|
||||
"proxima.stratified.cluster.first_class";
|
||||
"zvec.stratified.cluster.first_class";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_CLASS =
|
||||
"proxima.stratified.cluster.second_class";
|
||||
"zvec.stratified.cluster.second_class";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_COUNT =
|
||||
"proxima.stratified.cluster.first_count";
|
||||
"zvec.stratified.cluster.first_count";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_COUNT =
|
||||
"proxima.stratified.cluster.second_count";
|
||||
"zvec.stratified.cluster.second_count";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_PARAMS =
|
||||
"proxima.stratified.cluster.first_params";
|
||||
"zvec.stratified.cluster.first_params";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_PARAMS =
|
||||
"proxima.stratified.cluster.second_params";
|
||||
"zvec.stratified.cluster.second_params";
|
||||
static const std::string STRATIFIED_CLUSTER_AUTO_TUNING =
|
||||
"proxima.stratified.cluster.auto_tuning";
|
||||
"zvec.stratified.cluster.auto_tuning";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_POOL_COUNT =
|
||||
"proxima.stratified.cluster.second_pool_count";
|
||||
"zvec.stratified.cluster.second_pool_count";
|
||||
|
||||
//! Gap Statistics
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN =
|
||||
"proxima.gapstats.cluster_estimater.k_min";
|
||||
"zvec.gapstats.cluster_estimater.k_min";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX =
|
||||
"proxima.gapstats.cluster_estimater.k_max";
|
||||
"zvec.gapstats.cluster_estimater.k_max";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN_STEP =
|
||||
"proxima.gapstats.cluster_estimater.k_min_step";
|
||||
"zvec.gapstats.cluster_estimater.k_min_step";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX_STEP =
|
||||
"proxima.gapstats.cluster_estimater.k_max_step";
|
||||
"zvec.gapstats.cluster_estimater.k_max_step";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_TRY_COUNT =
|
||||
"proxima.gapstats.cluster_estimater.try_count";
|
||||
"zvec.gapstats.cluster_estimater.try_count";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_SHARD_FACTOR =
|
||||
"proxima.gapstats.cluster_estimater.shard_factor";
|
||||
"zvec.gapstats.cluster_estimater.shard_factor";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_ENABLE_MC2 =
|
||||
"proxima.gapstats.cluster_estimater.enable_mc2";
|
||||
"zvec.gapstats.cluster_estimater.enable_mc2";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.gapstats.cluster_estimater.markov_chain_length";
|
||||
"zvec.gapstats.cluster_estimater.markov_chain_length";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_CLUSTER_CLASS =
|
||||
"proxima.gapstats.cluster_estimater.cluster_class";
|
||||
"zvec.gapstats.cluster_estimater.cluster_class";
|
||||
|
||||
static const std::string CLUSTER_TRAINER_SAMPLE_COUNT =
|
||||
"proxima.cluster.trainer.sample_count";
|
||||
"zvec.cluster.trainer.sample_count";
|
||||
static const std::string CLUSTER_TRAINER_SAMPLE_RATIO =
|
||||
"proxima.cluster.trainer.sample_ratio";
|
||||
"zvec.cluster.trainer.sample_ratio";
|
||||
static const std::string CLUSTER_TRAINER_THREAD_COUNT =
|
||||
"proxima.cluster.trainer.thread_count";
|
||||
"zvec.cluster.trainer.thread_count";
|
||||
static const std::string CLUSTER_TRAINER_FILE_NAME =
|
||||
"proxima.cluster.trainer.file_name";
|
||||
"zvec.cluster.trainer.file_name";
|
||||
static const std::string CLUSTER_TRAINER_CLASS_NAME =
|
||||
"proxima.cluster.trainer.class_name";
|
||||
"zvec.cluster.trainer.class_name";
|
||||
|
||||
static const std::string STRATIFIED_TRAINER_SAMPLE_COUNT =
|
||||
"proxima.stratified.trainer.sample_count";
|
||||
"zvec.stratified.trainer.sample_count";
|
||||
static const std::string STRATIFIED_TRAINER_SAMPLE_RATIO =
|
||||
"proxima.stratified.trainer.sample_ratio";
|
||||
"zvec.stratified.trainer.sample_ratio";
|
||||
static const std::string STRATIFIED_TRAINER_THREAD_COUNT =
|
||||
"proxima.stratified.trainer.thread_count";
|
||||
"zvec.stratified.trainer.thread_count";
|
||||
static const std::string STRATIFIED_TRAINER_FILE_NAME =
|
||||
"proxima.stratified.trainer.file_name";
|
||||
"zvec.stratified.trainer.file_name";
|
||||
static const std::string STRATIFIED_TRAINER_CLASS_NAME =
|
||||
"proxima.stratified.trainer.class_name";
|
||||
"zvec.stratified.trainer.class_name";
|
||||
static const std::string STRATIFIED_TRAINER_CLUSTER_COUNT =
|
||||
"proxima.stratified.trainer.cluster_count";
|
||||
"zvec.stratified.trainer.cluster_count";
|
||||
static const std::string STRATIFIED_TRAINER_AUTOAUNE =
|
||||
"proxima.stratified.trainer.autotune";
|
||||
"zvec.stratified.trainer.autotune";
|
||||
static const std::string STRATIFIED_TRAINER_PARAMS_IN_LEVEL_PREFIX =
|
||||
"proxima.stratified.trainer.cluster_params_in_level_";
|
||||
"zvec.stratified.trainer.cluster_params_in_level_";
|
||||
|
||||
static const std::string MULTI_CHUNK_CLUSTER_COUNT =
|
||||
"zvec.cluster.multi_chunk_cluster.count";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_CHUNK_COUNT =
|
||||
"zvec.cluster.multi_chunk_cluster.chunk_count";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_THREAD_COUNT =
|
||||
"zvec.cluster.multi_chunk_cluster.thread_count";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_EPSILON =
|
||||
"zvec.cluster.multi_chunk_cluster.epsilon";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_MAX_ITERATIONS =
|
||||
"zvec.cluster.multi_chunk_cluster.max_iterations";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"zvec.cluster.multi_chunk_cluster.markov_chain_length";
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "multi_chunk_cluster.h"
|
||||
#include <chrono>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
bool MultiChunkClusterAlgorithm::is_valid(void) const {
|
||||
return features_ && features_->count();
|
||||
}
|
||||
|
||||
bool MultiChunkClusterAlgorithm::check_centroids(
|
||||
const IndexCluster::CentroidList & /*cents*/) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void MultiChunkClusterAlgorithm::init_centroids(
|
||||
size_t count, IndexCluster::CentroidList *out) {
|
||||
// Just resize, because the get random centroid step is done by cluster_once
|
||||
out->resize(count);
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::reset(void) {
|
||||
features_.reset();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::cleanup(void) {
|
||||
features_.reset();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void MultiChunkClusterAlgorithm::suggest(uint32_t k) {
|
||||
cluster_count_ = k;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::update(const ailego::Params ¶ms) {
|
||||
this->update_params(params);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! MultiChunkCluster
|
||||
int MultiChunkClusterAlgorithm::update_params(const ailego::Params ¶ms) {
|
||||
params.get(GENERAL_THREAD_COUNT, &thread_count_);
|
||||
params.get(GENERAL_CLUSTER_COUNT, &cluster_count_);
|
||||
|
||||
params.get(MULTI_CHUNK_CLUSTER_THREAD_COUNT, &thread_count_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_COUNT, &cluster_count_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_CHUNK_COUNT, &chunk_count_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_EPSILON, &epsilon_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_MAX_ITERATIONS, &max_iterations_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_MARKOV_CHAIN_LENGTH, &markov_chain_length_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::init_distance_func() {
|
||||
IndexMetric::Pointer metric_{};
|
||||
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("Create metric %s failed.", meta_.metric_name().c_str());
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
int ret = metric_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed wit ret %d.", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
distance_func_ = metric_->distance_matrix(1, 1);
|
||||
if (!distance_func_) {
|
||||
LOG_ERROR("DistanceMatrix function is nullptr.");
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::do_chunk() {
|
||||
if (chunk_count_ == 0) {
|
||||
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
size_t large_chunk_count = meta_.dimension() % chunk_count_;
|
||||
size_t base_chunk_dim_ = meta_.dimension() / chunk_count_;
|
||||
|
||||
chunk_dims_.clear();
|
||||
|
||||
for (size_t i = 0; i < chunk_count_; ++i) {
|
||||
if (i < large_chunk_count) {
|
||||
chunk_dims_.push_back(base_chunk_dim_ + 1);
|
||||
} else {
|
||||
chunk_dims_.push_back(base_chunk_dim_);
|
||||
}
|
||||
}
|
||||
|
||||
chunk_dim_offsets_.clear();
|
||||
chunk_dim_offsets_.push_back(0);
|
||||
for (size_t i = 1; i < chunk_count_; ++i) {
|
||||
chunk_dim_offsets_.push_back(chunk_dim_offsets_[i - 1] +
|
||||
chunk_dims_[i - 1]);
|
||||
}
|
||||
chunk_dim_offsets_.push_back(meta_.dimension());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
meta_ = meta;
|
||||
|
||||
int ret = update_params(params);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = do_chunk();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = init_distance_func();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::mount(IndexFeatures::Pointer features) {
|
||||
if (!features) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!features->is_matched(meta_)) {
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
auto data_type = meta_.data_type();
|
||||
if (data_type != IndexMeta::DataType::DT_FP32 &&
|
||||
data_type != IndexMeta::DataType::DT_FP16) {
|
||||
LOG_ERROR("Unsupported meta type %u", data_type);
|
||||
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
features_ = std::move(features);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! cluster
|
||||
int MultiChunkClusterAlgorithm::cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
if (chunk_count_ == 0) {
|
||||
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (cluster_count_ == 0) {
|
||||
LOG_ERROR("Invalid cluster Count: %u", cluster_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
}
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
cents.clear();
|
||||
cents.resize(chunk_count_ * cluster_count_);
|
||||
|
||||
std::atomic<size_t> finished{0};
|
||||
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(
|
||||
ailego::Closure::New(this, &MultiChunkClusterAlgorithm::do_cluster, i,
|
||||
threads->count(), ¢s, &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Finish Chunk Count %zu, Finished Percent %.3f%%", finished.load(),
|
||||
finished.load() * 100.0f / chunk_count_);
|
||||
}
|
||||
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Classify
|
||||
int MultiChunkClusterAlgorithm::classify(
|
||||
IndexThreads::Pointer /*threads*/, IndexCluster::CentroidList & /*cents*/) {
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
//! Label
|
||||
int MultiChunkClusterAlgorithm::label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out) {
|
||||
if (chunk_count_ == 0) {
|
||||
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (cents.empty()) {
|
||||
LOG_ERROR("The input centroid's list is empty.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->check_centroids(cents)) {
|
||||
LOG_ERROR("The input centroid's list includes some invalid centroids.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->is_valid()) {
|
||||
LOG_ERROR("The cluster is not ready.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (cluster_count_ == 0) {
|
||||
LOG_ERROR("Invalid cluster Count: %u", cluster_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
size_t features_count = features_->count();
|
||||
out->resize(features_count * chunk_count_);
|
||||
|
||||
std::atomic<size_t> finished{0};
|
||||
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(
|
||||
ailego::Closure::New(this, &MultiChunkClusterAlgorithm::do_label, i,
|
||||
threads->count(), cents, out, &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Finish label cnt %zu, finished percent %.3f%%", finished.load(),
|
||||
finished.load() * 100.0f / features_count);
|
||||
}
|
||||
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cluster
|
||||
int MultiChunkCluster::cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
return algorithm_->cluster(std::move(threads), cents);
|
||||
}
|
||||
|
||||
//! Classify
|
||||
int MultiChunkCluster::classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
return algorithm_->classify(std::move(threads), cents);
|
||||
}
|
||||
|
||||
//! Label
|
||||
int MultiChunkCluster::label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out) {
|
||||
return algorithm_->label(std::move(threads), cents, out);
|
||||
}
|
||||
|
||||
//! Update Cluster
|
||||
int MultiChunkCluster::update(const ailego::Params ¶ms) {
|
||||
return algorithm_->update(params);
|
||||
}
|
||||
|
||||
//! Reset Cluster
|
||||
int MultiChunkCluster::reset(void) {
|
||||
return algorithm_->reset();
|
||||
}
|
||||
|
||||
//! Cleanup Cluster
|
||||
int MultiChunkCluster::cleanup(void) {
|
||||
return algorithm_->cleanup();
|
||||
}
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
void MultiChunkCluster::suggest(uint32_t k) {
|
||||
algorithm_->suggest(k);
|
||||
}
|
||||
|
||||
int MultiChunkCluster::mount(IndexFeatures::Pointer feats) {
|
||||
return algorithm_->mount(feats);
|
||||
}
|
||||
|
||||
int MultiChunkCluster::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
IndexMeta new_meta(meta.data_type(), meta.dimension());
|
||||
|
||||
if (meta.metric_name() == "Cosine") {
|
||||
new_meta.set_dimension(meta.dimension() - 1);
|
||||
new_meta.set_metric("InnerProduct", 0, ailego::Params());
|
||||
}
|
||||
|
||||
auto data_type = new_meta.data_type();
|
||||
|
||||
if (new_meta.metric_name() == "InnerProduct") {
|
||||
switch (data_type) {
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
algorithm_.reset(
|
||||
new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<ailego::Float16>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<float>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP64: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<double>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT8: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<int8_t>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT16: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<int16_t>);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
LOG_ERROR("Unsupported feature types %d.", data_type);
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
switch (data_type) {
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<ailego::Float16>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<float>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP64: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<double>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT8: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<int8_t>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT16: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<int16_t>);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
LOG_ERROR("Unsupported feature types %d.", data_type);
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
algorithm_->init(new_meta, params);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,466 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/algorithm/kmeans.h>
|
||||
#include <ailego/parallel/multi_thread_list.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include <zvec/core/framework/index_cluster.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/framework/index_factory.h>
|
||||
#include "cluster_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! MultiChunkClusterAlgorithm
|
||||
class MultiChunkClusterAlgorithm {
|
||||
public:
|
||||
typedef std::shared_ptr<MultiChunkClusterAlgorithm> Pointer;
|
||||
|
||||
//! Constructor
|
||||
MultiChunkClusterAlgorithm() = default;
|
||||
|
||||
//! Destructor
|
||||
virtual ~MultiChunkClusterAlgorithm() = default;
|
||||
|
||||
//! Initialize Cluster
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms);
|
||||
|
||||
//! Cleanup Cluster
|
||||
int cleanup();
|
||||
|
||||
//! Reset Cluster
|
||||
int reset();
|
||||
|
||||
//! Update Cluster
|
||||
int update(const ailego::Params ¶ms);
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
void suggest(uint32_t k);
|
||||
|
||||
//! Mount features
|
||||
int mount(IndexFeatures::Pointer feats);
|
||||
|
||||
//! Cluster
|
||||
int cluster(IndexThreads::Pointer threads, IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Classify
|
||||
int classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Label
|
||||
int label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out);
|
||||
|
||||
|
||||
const std::vector<uint32_t> &chunk_dims() const {
|
||||
return chunk_dims_;
|
||||
}
|
||||
|
||||
const std::vector<uint32_t> &chunk_dim_offsets() const {
|
||||
return chunk_dim_offsets_;
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Check Centroids
|
||||
bool check_centroids(const IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Test if it is valid
|
||||
bool is_valid() const;
|
||||
|
||||
//! Do chunk
|
||||
int do_chunk();
|
||||
|
||||
//! Update parameters
|
||||
int update_params(const ailego::Params ¶ms);
|
||||
|
||||
int init_distance_func();
|
||||
|
||||
//! cluster thread
|
||||
virtual void do_cluster(size_t idx, size_t chunk_step,
|
||||
IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished) = 0;
|
||||
|
||||
//! label thread
|
||||
virtual void do_label(size_t idx, size_t step,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out,
|
||||
std::atomic<size_t> *finished) = 0;
|
||||
|
||||
//! Initialize Centroids
|
||||
void init_centroids(size_t count, IndexCluster::CentroidList *out);
|
||||
|
||||
private:
|
||||
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
|
||||
|
||||
protected:
|
||||
uint32_t cluster_count_{0u};
|
||||
uint32_t thread_count_{0u};
|
||||
uint32_t chunk_count_{0u};
|
||||
uint32_t max_iterations_{20u};
|
||||
bool assumption_free_{false};
|
||||
uint32_t markov_chain_length_{32};
|
||||
double epsilon_{std::numeric_limits<float>::epsilon()};
|
||||
|
||||
int errcode_{0};
|
||||
std::atomic_bool error_{false};
|
||||
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
|
||||
std::mutex mutex_{};
|
||||
std::condition_variable cond_{};
|
||||
|
||||
IndexMeta meta_{};
|
||||
IndexFeatures::Pointer features_{};
|
||||
|
||||
std::vector<uint32_t> chunk_dims_;
|
||||
std::vector<uint32_t> chunk_dim_offsets_;
|
||||
|
||||
IndexMetric::MatrixDistance distance_func_{nullptr};
|
||||
};
|
||||
|
||||
/*! Numerical cluster algorithm
|
||||
*/
|
||||
template <typename T>
|
||||
class MultiChunkNumericalAlgorithm : public MultiChunkClusterAlgorithm {
|
||||
public:
|
||||
//! Type of value
|
||||
using ValueType = typename std::remove_cv<T>::type;
|
||||
|
||||
// Check supporting type
|
||||
static_assert(ailego::IsArithmetic<ValueType>::value,
|
||||
"ValueType must be arithmetic");
|
||||
|
||||
//! Constructor
|
||||
MultiChunkNumericalAlgorithm() = default;
|
||||
|
||||
//! Destructor
|
||||
~MultiChunkNumericalAlgorithm() = default;
|
||||
|
||||
protected:
|
||||
//! cluster thread
|
||||
void do_cluster(size_t idx, size_t chunk_step,
|
||||
IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished);
|
||||
|
||||
//! label thread
|
||||
void do_label(size_t idx, size_t step,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished);
|
||||
};
|
||||
|
||||
//! cluster thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalAlgorithm<T>::do_cluster(
|
||||
size_t idx, size_t chunk_step, IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished) {
|
||||
for (size_t chunk = idx; chunk < chunk_count_; chunk += chunk_step) {
|
||||
auto chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
ailego::NumericalKmeans<T, IndexThreads> algorithm(cluster_count_,
|
||||
chunk_dim);
|
||||
|
||||
// mount features into algorithm
|
||||
auto features_count = features_->count();
|
||||
|
||||
algorithm.feature_matrix_reserve(features_count);
|
||||
|
||||
for (size_t i = 0; i < features_count; ++i) {
|
||||
auto vec = reinterpret_cast<const T *>(features_->element(i));
|
||||
algorithm.append(vec + chunk_dim_offsets_[chunk], chunk_dim);
|
||||
}
|
||||
|
||||
IndexThreads::Pointer local_threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(1, false);
|
||||
if (!local_threads) {
|
||||
error_ = IndexError_NoMemory;
|
||||
return;
|
||||
}
|
||||
|
||||
ailego::Kmc2CentroidsGenerator<ailego::NumericalKmeans<T, IndexThreads>,
|
||||
IndexThreads>
|
||||
cent_gen;
|
||||
cent_gen.set_chain_length(markov_chain_length_);
|
||||
cent_gen.set_assumption_free(assumption_free_);
|
||||
cent_gen(&algorithm, *local_threads);
|
||||
|
||||
double cost = 0.0;
|
||||
|
||||
for (uint32_t i = 0; i < max_iterations_; ++i) {
|
||||
double old_cost, new_epsilon;
|
||||
old_cost = cost;
|
||||
|
||||
bool result = algorithm.cluster_once(*local_threads, &cost);
|
||||
if (result != true) {
|
||||
LOG_ERROR("(%u) Failed to cluster.", i + 1);
|
||||
errcode_ = -1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
new_epsilon = std::abs(cost - old_cost);
|
||||
if (new_epsilon < epsilon_) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto &chunk_cents = algorithm.centroids();
|
||||
|
||||
for (size_t i = 0; i < chunk_cents.count(); ++i) {
|
||||
size_t global_cent_idx = chunk * cluster_count_ + i;
|
||||
|
||||
IndexCluster::Centroid *centroid = &(cents->at(global_cent_idx));
|
||||
centroid->set_score(algorithm.context().clusters()[i].cost());
|
||||
centroid->set_follows(algorithm.context().clusters()[i].count());
|
||||
centroid->set_feature(algorithm.centroids()[i],
|
||||
chunk_dim * meta_.unit_size());
|
||||
}
|
||||
|
||||
LOG_INFO("(%zu) Chunk Done. Clusters Count: %zu, Features: %zu, Cost: %f",
|
||||
chunk, algorithm.centroids().count(), features_->count(), cost);
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
//! label thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalAlgorithm<T>::do_label(
|
||||
size_t idx, size_t step, const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished) {
|
||||
for (size_t id = idx; id < features_->count(); id += step) {
|
||||
const T *feat = reinterpret_cast<const T *>(features_->element(id));
|
||||
|
||||
for (size_t chunk = 0; chunk < chunk_count_; ++chunk) {
|
||||
size_t chunk_dim_offset = chunk_dim_offsets_[chunk];
|
||||
size_t chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
uint32_t sel_index = 0;
|
||||
float sel_score = std::numeric_limits<float>::max();
|
||||
|
||||
for (uint32_t cluster = 0; cluster < cluster_count_; ++cluster) {
|
||||
float score{0.0};
|
||||
|
||||
distance_func_(cents[chunk * cluster_count_ + cluster].feature(),
|
||||
feat + chunk_dim_offset, chunk_dim, &score);
|
||||
|
||||
if (score < sel_score) {
|
||||
sel_score = score;
|
||||
sel_index = cluster;
|
||||
}
|
||||
}
|
||||
|
||||
(*out)[id * chunk_count_ + chunk] = static_cast<uint32_t>(sel_index);
|
||||
}
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
/*! Inner Product Cluster Algorithm
|
||||
*/
|
||||
template <typename T>
|
||||
class MultiChunkNumericalInnerProductAlgorithm
|
||||
: public MultiChunkClusterAlgorithm {
|
||||
public:
|
||||
//! Type of value
|
||||
using ValueType = typename std::remove_cv<T>::type;
|
||||
|
||||
// Check supporting type
|
||||
static_assert(ailego::IsArithmetic<ValueType>::value,
|
||||
"ValueType must be arithmetic");
|
||||
|
||||
//! Constructor
|
||||
MultiChunkNumericalInnerProductAlgorithm() = default;
|
||||
|
||||
//! Destructor
|
||||
~MultiChunkNumericalInnerProductAlgorithm() = default;
|
||||
|
||||
protected:
|
||||
//! cluster thread
|
||||
void do_cluster(size_t idx, size_t chunk_step,
|
||||
IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished);
|
||||
|
||||
//! label thread
|
||||
void do_label(size_t idx, size_t chunk_step,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished);
|
||||
};
|
||||
|
||||
//! cluster thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalInnerProductAlgorithm<T>::do_cluster(
|
||||
size_t idx, size_t chunk_step, IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished) {
|
||||
for (size_t chunk = idx; chunk < chunk_count_; chunk += chunk_step) {
|
||||
auto chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
ailego::NumericalInnerProductKmeans<T, IndexThreads> algorithm(
|
||||
cluster_count_, chunk_dim);
|
||||
|
||||
// mount features into algorithm
|
||||
auto features_count = features_->count();
|
||||
|
||||
algorithm.feature_matrix_reserve(features_count);
|
||||
|
||||
for (size_t i = 0; i < features_count; ++i) {
|
||||
auto vec = reinterpret_cast<const T *>(features_->element(i));
|
||||
algorithm.append(vec + chunk_dim_offsets_[chunk], chunk_dim);
|
||||
}
|
||||
|
||||
IndexThreads::Pointer local_threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(1, false);
|
||||
if (!local_threads) {
|
||||
error_ = IndexError_NoMemory;
|
||||
return;
|
||||
}
|
||||
|
||||
ailego::Kmc2CentroidsGenerator<
|
||||
ailego::NumericalInnerProductKmeans<T, IndexThreads>, IndexThreads>
|
||||
cent_gen;
|
||||
cent_gen.set_chain_length(markov_chain_length_);
|
||||
cent_gen.set_assumption_free(assumption_free_);
|
||||
cent_gen(&algorithm, *local_threads);
|
||||
|
||||
double cost = 0.0;
|
||||
|
||||
for (uint32_t i = 0; i < max_iterations_; ++i) {
|
||||
double old_cost, new_epsilon;
|
||||
old_cost = cost;
|
||||
|
||||
bool result = algorithm.cluster_once(*local_threads, &cost);
|
||||
if (result != true) {
|
||||
LOG_ERROR("(%zu) Failed to cluster.", (size_t)(i + 1));
|
||||
errcode_ = -1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
new_epsilon = std::abs(cost - old_cost);
|
||||
if (new_epsilon < epsilon_) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto &chunk_cents = algorithm.centroids();
|
||||
|
||||
for (size_t i = 0; i < chunk_cents.count(); ++i) {
|
||||
size_t global_cent_idx = chunk * cluster_count_ + i;
|
||||
|
||||
IndexCluster::Centroid *centroid = &(cents->at(global_cent_idx));
|
||||
centroid->set_score(algorithm.context().clusters()[i].cost());
|
||||
centroid->set_follows(algorithm.context().clusters()[i].count());
|
||||
centroid->set_feature(algorithm.centroids()[i],
|
||||
chunk_dim * meta_.unit_size());
|
||||
}
|
||||
|
||||
LOG_INFO("(%zu) Chunk Done. Clusters Count: %zu, Features: %zu, Cost: %f",
|
||||
chunk, algorithm.centroids().count(), features_->count(), cost);
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
//! label thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalInnerProductAlgorithm<T>::do_label(
|
||||
size_t idx, size_t step, const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished) {
|
||||
for (size_t id = idx; id < features_->count(); id += step) {
|
||||
const T *feat = reinterpret_cast<const T *>(features_->element(id));
|
||||
|
||||
for (size_t chunk = 0; chunk < chunk_count_; ++chunk) {
|
||||
size_t chunk_dim_offset = chunk_dim_offsets_[chunk];
|
||||
size_t chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
uint32_t sel_index = 0;
|
||||
float sel_score = std::numeric_limits<float>::max();
|
||||
|
||||
for (uint32_t cluster = 0; cluster < cluster_count_; ++cluster) {
|
||||
float score{0.0};
|
||||
|
||||
distance_func_(cents[chunk * cluster_count_ + cluster].feature(),
|
||||
feat + chunk_dim_offset, chunk_dim, &score);
|
||||
|
||||
if (score < sel_score) {
|
||||
sel_score = score;
|
||||
sel_index = cluster;
|
||||
}
|
||||
}
|
||||
|
||||
(*out)[id * chunk_count_ + chunk] = sel_index;
|
||||
}
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
//! MultiChunkCluster
|
||||
class MultiChunkCluster {
|
||||
public:
|
||||
std::shared_ptr<MultiChunkCluster> Pointer;
|
||||
|
||||
//! Constructor
|
||||
MultiChunkCluster() = default;
|
||||
|
||||
//! Destructor
|
||||
~MultiChunkCluster() = default;
|
||||
|
||||
//! Initialize Cluster
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms);
|
||||
|
||||
//! Cleanup Cluster
|
||||
int cleanup();
|
||||
|
||||
//! Reset Cluster
|
||||
int reset();
|
||||
|
||||
//! Update Cluster
|
||||
int update(const ailego::Params ¶ms);
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
void suggest(uint32_t k);
|
||||
|
||||
//! Mount features
|
||||
int mount(IndexFeatures::Pointer feats);
|
||||
|
||||
//! Cluster
|
||||
int cluster(IndexThreads::Pointer threads, IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Classify
|
||||
int classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Label
|
||||
int label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out);
|
||||
|
||||
const std::vector<uint32_t> &chunk_dims() const {
|
||||
return algorithm_->chunk_dims();
|
||||
}
|
||||
|
||||
const std::vector<uint32_t> &chunk_dim_offsets() const {
|
||||
return algorithm_->chunk_dim_offsets();
|
||||
}
|
||||
|
||||
protected:
|
||||
MultiChunkClusterAlgorithm::Pointer algorithm_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,57 @@
|
||||
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
|
||||
|
||||
file(GLOB_RECURSE ALL_SRCS *.cc *.c)
|
||||
|
||||
# The DiskAnn plugin is loaded at runtime via zvec::LoadDiskAnnPlugin() with
|
||||
# RTLD_GLOBAL, so its undefined references to internal zvec symbols
|
||||
# (core_framework, core_knn_cluster, zvec_ailego, ...) are resolved at load
|
||||
# time against the hosting binary (_zvec.so for the Python extension, the
|
||||
# test executable for gtest, or libzvec_core for tools).
|
||||
#
|
||||
# As a consequence the plugin .so must NOT carry NEEDED entries for those
|
||||
# libs: otherwise the dynamic loader would try to resolve them from disk
|
||||
# before invoking dlopen's RTLD_GLOBAL symbol sharing, and the Python wheel
|
||||
# (which only ships _zvec.so + the plugin) would fail to load.
|
||||
#
|
||||
# We therefore link the plugin only against its system-level dependency
|
||||
# (libaio) and rely on the global include dirs plus -Wl,--unresolved-symbols
|
||||
# =ignore-all to let the linker build a shared library with unresolved
|
||||
# references to internal APIs.
|
||||
set(CORE_KNN_DISKANN_LIBS "")
|
||||
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386")
|
||||
list(APPEND CORE_KNN_DISKANN_LIBS aio)
|
||||
endif()
|
||||
|
||||
cc_library(
|
||||
NAME core_knn_diskann
|
||||
SHARED STRICT
|
||||
SRCS *.cc
|
||||
LIBS ${CORE_KNN_DISKANN_LIBS}
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
|
||||
# Internal zvec libs are referenced only for header availability; the actual
|
||||
# symbol resolution happens at dlopen time. Expose their public include dirs
|
||||
# without adding them as link dependencies.
|
||||
foreach(_dep zvec_ailego core_framework core_knn_cluster)
|
||||
if(TARGET ${_dep})
|
||||
target_include_directories(core_knn_diskann PRIVATE
|
||||
$<TARGET_PROPERTY:${_dep},INTERFACE_INCLUDE_DIRECTORIES>)
|
||||
add_dependencies(core_knn_diskann ${_dep})
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
# Allow the plugin to have unresolved symbols that will be satisfied at
|
||||
# dlopen(RTLD_NOW | RTLD_GLOBAL) time by the hosting binary.
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
target_link_options(core_knn_diskann PRIVATE
|
||||
"LINKER:--unresolved-symbols=ignore-all")
|
||||
endif()
|
||||
|
||||
# Rename the artifact to libzvec_diskann_plugin.so so it is clearly identified
|
||||
# as an optional plugin that users load at runtime via zvec::LoadDiskAnnPlugin().
|
||||
set_target_properties(core_knn_diskann PROPERTIES
|
||||
OUTPUT_NAME zvec_diskann_plugin
|
||||
)
|
||||
@@ -0,0 +1,324 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_algorithm.h"
|
||||
#include <algorithm>
|
||||
#include <set>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnAlgorithm::DiskAnnAlgorithm(DiskAnnEntity &entity, uint32_t max_degree)
|
||||
: entity_(entity), max_degree_(max_degree), lock_pool_(kLockCnt) {}
|
||||
|
||||
std::vector<diskann_id_t> DiskAnnAlgorithm::get_init_ids(DiskAnnContext *ctx) {
|
||||
const auto &entity = ctx->get_entity();
|
||||
|
||||
std::vector<diskann_id_t> init_ids;
|
||||
|
||||
init_ids.emplace_back(entity.medoid());
|
||||
|
||||
return init_ids;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::add_node(diskann_id_t id, DiskAnnContext *ctx) {
|
||||
const void *vec = entity_.get_vector(id);
|
||||
|
||||
ctx->reset_query(vec);
|
||||
|
||||
std::vector<diskann_id_t> pruned_list;
|
||||
|
||||
int ret = search_neighbor_and_prune(id, pruned_list, ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
{
|
||||
auto lock = lock_for(id);
|
||||
entity_.set_neighbors(id, pruned_list);
|
||||
}
|
||||
|
||||
return inter_insert(id, pruned_list, ctx);
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::prune_node(diskann_id_t id, DiskAnnContext *ctx) {
|
||||
DistCalculator &dc = ctx->dist_calculator();
|
||||
|
||||
auto neighbors = entity_.get_neighbors(id);
|
||||
|
||||
if (neighbors.first > max_degree_) {
|
||||
std::set<diskann_id_t> dummy_visited;
|
||||
std::vector<Neighbor> dummy_pool(0);
|
||||
std::vector<diskann_id_t> new_out_neighbors;
|
||||
|
||||
for (size_t i = 0; i < neighbors.first; ++i) {
|
||||
diskann_id_t node_id = (neighbors.second)[i];
|
||||
|
||||
auto itr = dummy_visited.find(node_id);
|
||||
if (itr == dummy_visited.end() && node_id != id) {
|
||||
float dist = dc.dist(id, node_id);
|
||||
|
||||
dummy_pool.emplace_back(Neighbor(node_id, dist));
|
||||
dummy_visited.insert(node_id);
|
||||
}
|
||||
}
|
||||
|
||||
prune_neighbors(id, dummy_pool, new_out_neighbors, ctx);
|
||||
|
||||
{
|
||||
auto lock = lock_for(id);
|
||||
entity_.set_neighbors(id, new_out_neighbors);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::inter_insert(diskann_id_t id,
|
||||
std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx) {
|
||||
DistCalculator &dc = ctx->dist_calculator();
|
||||
|
||||
for (auto &des : pruned_list) {
|
||||
std::vector<diskann_id_t> new_neighbors;
|
||||
bool need_prune = false;
|
||||
|
||||
{
|
||||
auto lock = lock_for(des);
|
||||
|
||||
auto neighbors = entity_.get_neighbors(des);
|
||||
|
||||
bool found = false;
|
||||
for (size_t i = 0; i < neighbors.first; ++i) {
|
||||
if ((neighbors.second)[i] == id) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
if (neighbors.first <
|
||||
static_cast<uint64_t>(DiskAnnEntity::kDefaultGraphSlackFactor *
|
||||
max_degree_)) {
|
||||
entity_.add_neighbor(des, id);
|
||||
need_prune = false;
|
||||
} else {
|
||||
new_neighbors.resize(neighbors.first + 1);
|
||||
memcpy(&new_neighbors[0], neighbors.second,
|
||||
sizeof(diskann_id_t) * neighbors.first);
|
||||
|
||||
new_neighbors[neighbors.first] = id;
|
||||
|
||||
need_prune = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (need_prune) {
|
||||
std::set<diskann_id_t> new_visited;
|
||||
std::vector<Neighbor> new_pool(0);
|
||||
|
||||
size_t reserve_size = static_cast<size_t>(std::ceil(
|
||||
1.05 * DiskAnnEntity::kDefaultGraphSlackFactor * max_degree_));
|
||||
|
||||
new_pool.reserve(reserve_size);
|
||||
|
||||
for (auto node_id : new_neighbors) {
|
||||
if (new_visited.find(node_id) == new_visited.end() && node_id != des) {
|
||||
float dist = dc.dist(des, node_id);
|
||||
new_pool.emplace_back(Neighbor(node_id, dist));
|
||||
new_visited.insert(node_id);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<diskann_id_t> new_pruned_neighbors;
|
||||
prune_neighbors(des, new_pool, new_pruned_neighbors, ctx);
|
||||
|
||||
{
|
||||
auto lock = lock_for(des);
|
||||
entity_.set_neighbors(des, new_pruned_neighbors);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::iterate_to_fixed_point(
|
||||
const std::vector<diskann_id_t> &init_ids, DiskAnnContext *ctx) {
|
||||
DistCalculator &dc = ctx->dist_calculator();
|
||||
std::vector<Neighbor> &expanded_nodes = ctx->expanded_nodes();
|
||||
NeighborPriorityQueue &best_list_nodes = ctx->best_list_nodes();
|
||||
VisitFilter &visit = ctx->visit_filter();
|
||||
|
||||
best_list_nodes.reserve(ctx->list_size());
|
||||
|
||||
for (auto id : init_ids) {
|
||||
const void *vec = entity_.get_vector(id);
|
||||
|
||||
float distance = dc.dist(vec);
|
||||
|
||||
Neighbor nn = Neighbor(id, distance);
|
||||
best_list_nodes.insert(nn);
|
||||
}
|
||||
|
||||
while (best_list_nodes.has_unexpanded_node()) {
|
||||
auto neighbor = best_list_nodes.closest_unexpanded();
|
||||
auto node_id = neighbor.id;
|
||||
|
||||
expanded_nodes.emplace_back(neighbor);
|
||||
|
||||
std::vector<diskann_id_t> id_scratch;
|
||||
{
|
||||
auto lock = lock_for(node_id);
|
||||
auto neighbors = entity_.get_neighbors(node_id);
|
||||
|
||||
for (size_t i = 0; i < neighbors.first; ++i) {
|
||||
diskann_id_t neighbor_id = (neighbors.second)[i];
|
||||
|
||||
if (!visit.visited(neighbor_id)) {
|
||||
id_scratch.push_back(neighbor_id);
|
||||
|
||||
visit.set_visited(neighbor_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < id_scratch.size(); ++i) {
|
||||
diskann_id_t id = id_scratch[i];
|
||||
|
||||
const void *vec = entity_.get_vector(id);
|
||||
float dist = dc.dist(vec);
|
||||
|
||||
best_list_nodes.insert(Neighbor(id, dist));
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::occlude_list(diskann_id_t id, std::vector<Neighbor> &pool,
|
||||
std::vector<diskann_id_t> &result,
|
||||
DiskAnnContext *ctx) {
|
||||
if (pool.size() == 0) return 0;
|
||||
|
||||
DistCalculator &dc = ctx->dist_calculator();
|
||||
|
||||
ailego_assert(std::is_sorted(pool.begin(), pool.end()));
|
||||
ailego_assert(result.size() == 0);
|
||||
|
||||
if (pool.size() > max_candidate_size_) {
|
||||
pool.resize(max_candidate_size_);
|
||||
}
|
||||
|
||||
std::vector<float> &occlude_factor = ctx->occlude_factor();
|
||||
|
||||
occlude_factor.clear();
|
||||
occlude_factor.insert(occlude_factor.end(), pool.size(), 0.0f);
|
||||
|
||||
float cur_alpha = 1;
|
||||
while (cur_alpha <= alpha_ && result.size() < max_degree_) {
|
||||
for (auto iter = pool.begin();
|
||||
result.size() < max_degree_ && iter != pool.end(); ++iter) {
|
||||
if (occlude_factor[iter - pool.begin()] > cur_alpha) {
|
||||
continue;
|
||||
}
|
||||
|
||||
occlude_factor[iter - pool.begin()] = std::numeric_limits<float>::max();
|
||||
|
||||
if (iter->id != id) {
|
||||
result.push_back(iter->id);
|
||||
}
|
||||
|
||||
for (auto iter2 = iter + 1; iter2 != pool.end(); iter2++) {
|
||||
auto t = iter2 - pool.begin();
|
||||
if (occlude_factor[t] > alpha_) {
|
||||
continue;
|
||||
}
|
||||
|
||||
float djk = dc.dist(iter2->id, iter->id);
|
||||
|
||||
occlude_factor[t] =
|
||||
(djk == 0) ? std::numeric_limits<float>::max()
|
||||
: std::max(occlude_factor[t], iter2->distance / djk);
|
||||
}
|
||||
}
|
||||
cur_alpha *= 1.2f;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::prune_neighbors(diskann_id_t id,
|
||||
std::vector<Neighbor> &pool,
|
||||
std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx) {
|
||||
if (pool.size() == 0) {
|
||||
pruned_list.clear();
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::sort(pool.begin(), pool.end());
|
||||
|
||||
pruned_list.clear();
|
||||
pruned_list.reserve(max_degree_);
|
||||
|
||||
occlude_list(id, pool, pruned_list, ctx);
|
||||
|
||||
ailego_assert(pruned_list.size() <= max_degree_);
|
||||
|
||||
if (saturate_graph_ && alpha_ > 1) {
|
||||
for (const auto &node : pool) {
|
||||
if (pruned_list.size() >= max_degree_) {
|
||||
break;
|
||||
}
|
||||
|
||||
if ((std::find(pruned_list.begin(), pruned_list.end(), node.id) ==
|
||||
pruned_list.end()) &&
|
||||
node.id != id) {
|
||||
pruned_list.push_back(node.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnAlgorithm::search_neighbor_and_prune(
|
||||
diskann_id_t id, std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx) {
|
||||
const std::vector<diskann_id_t> init_ids = get_init_ids(ctx);
|
||||
|
||||
int ret = iterate_to_fixed_point(init_ids, ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto &pool = ctx->expanded_nodes();
|
||||
pool.erase(std::remove_if(pool.begin(), pool.end(),
|
||||
[id](const auto &node) { return node.id == id; }),
|
||||
pool.end());
|
||||
|
||||
ret = prune_neighbors(id, pool, pruned_list, ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include <zvec/core/framework/index_meta.h>
|
||||
#include "diskann_context.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnAlgorithm {
|
||||
public:
|
||||
typedef std::unique_ptr<DiskAnnAlgorithm> UPointer;
|
||||
|
||||
public:
|
||||
DiskAnnAlgorithm(DiskAnnEntity &entity, uint32_t max_degree);
|
||||
|
||||
public:
|
||||
int add_node(diskann_id_t id, DiskAnnContext *ctx);
|
||||
int prune_node(diskann_id_t id, DiskAnnContext *ctx);
|
||||
|
||||
private:
|
||||
int search_neighbor_and_prune(diskann_id_t id,
|
||||
std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx);
|
||||
int iterate_to_fixed_point(const std::vector<diskann_id_t> &init_ids,
|
||||
DiskAnnContext *ctx);
|
||||
int prune_neighbors(diskann_id_t id, std::vector<Neighbor> &pool,
|
||||
std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx);
|
||||
int inter_insert(diskann_id_t id, std::vector<diskann_id_t> &pruned_list,
|
||||
DiskAnnContext *ctx);
|
||||
int occlude_list(diskann_id_t id, std::vector<Neighbor> &pool,
|
||||
std::vector<diskann_id_t> &result, DiskAnnContext *ctx);
|
||||
|
||||
std::vector<diskann_id_t> get_init_ids(DiskAnnContext *ctx);
|
||||
|
||||
private:
|
||||
static constexpr uint32_t kLockCnt{1U << 16};
|
||||
static constexpr uint32_t kLockMask{kLockCnt - 1U};
|
||||
|
||||
DiskAnnEntity &entity_;
|
||||
|
||||
uint32_t max_degree_{DiskAnnEntity::kDefaultMaxDegree};
|
||||
uint32_t max_candidate_size_{DiskAnnEntity::kDefaultMaxOcclusionSize};
|
||||
|
||||
std::vector<std::mutex> lock_pool_{};
|
||||
|
||||
//! Acquire the striped lock guarding the given id (RAII).
|
||||
[[nodiscard]] std::unique_lock<std::mutex> lock_for(diskann_id_t id) {
|
||||
return std::unique_lock<std::mutex>(lock_pool_[id & kLockMask]);
|
||||
}
|
||||
|
||||
float alpha_{DiskAnnEntity::kDefaultAlpha};
|
||||
bool saturate_graph_{true};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,630 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_builder.h"
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <ailego/math/euclidean_distance_matrix.h>
|
||||
#include <ailego/math/normalizer.h>
|
||||
#include <ailego/pattern/defer.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/interface/index_factory.h>
|
||||
#include "algorithm/cluster/vector_mean.h"
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int DiskAnnBuilder::init(const IndexMeta &meta, const ailego::Params ¶ms) {
|
||||
LOG_INFO("Begin DiskAnnBuilder::init");
|
||||
|
||||
params.get(PARAM_DISKANN_BUILDER_MAX_DEGREE, &max_degree_);
|
||||
params.get(PARAM_DISKANN_BUILDER_LIST_SIZE, &list_size_);
|
||||
params.get(PARAM_DISKANN_BUILDER_THREAD_COUNT, &build_thread_count_);
|
||||
|
||||
if (build_thread_count_ == 0) {
|
||||
build_thread_count_ = std::thread::hardware_concurrency();
|
||||
}
|
||||
|
||||
if (build_thread_count_ > std::thread::hardware_concurrency()) {
|
||||
LOG_WARN("Build thread count [%s] greater than cpu cores %u",
|
||||
PARAM_DISKANN_BUILDER_THREAD_COUNT.c_str(),
|
||||
std::thread::hardware_concurrency());
|
||||
}
|
||||
|
||||
uint32_t max_pq_chunk_num{0};
|
||||
if (params.get(PARAM_DISKANN_BUILDER_MAX_PQ_CHUNK_NUM, &max_pq_chunk_num)) {
|
||||
if (max_pq_chunk_num > meta.dimension()) {
|
||||
LOG_ERROR(
|
||||
"PQ Chunk Num larger than dimension, PQ Chunk Num: %d, Dimension: %d",
|
||||
max_pq_chunk_num, meta.dimension());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
max_pq_chunk_num_ = max_pq_chunk_num;
|
||||
}
|
||||
|
||||
if (params.has(PARAM_DISKANN_BUILDER_MEMORY_LIMIT)) {
|
||||
params.get(PARAM_DISKANN_BUILDER_MEMORY_LIMIT, &memory_limit_);
|
||||
if (memory_limit_ <= 0) {
|
||||
LOG_ERROR("Invalid memory limit: %lf", memory_limit_);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
memory_limit_set_ = true;
|
||||
}
|
||||
|
||||
if (params.has(PARAM_DISKANN_BUILDER_MAX_TRAIN_SAMPLE_COUNT)) {
|
||||
params.get(PARAM_DISKANN_BUILDER_MAX_TRAIN_SAMPLE_COUNT,
|
||||
&max_train_sample_count_);
|
||||
}
|
||||
|
||||
if (params.has(PARAM_DISKANN_BUILDER_TRAIN_SAMPLE_RATIO)) {
|
||||
params.get(PARAM_DISKANN_BUILDER_TRAIN_SAMPLE_RATIO, &train_sample_ratio_);
|
||||
}
|
||||
|
||||
raw_meta_ = meta;
|
||||
|
||||
build_meta_ = meta;
|
||||
if (meta.metric_name() == "InnerProduct") {
|
||||
build_meta_.set_metric("SquaredEuclidean", 0, ailego::Params());
|
||||
} else if (meta.metric_name() == "Cosine") {
|
||||
build_meta_.set_metric("SquaredEuclidean", 0, ailego::Params());
|
||||
|
||||
if (meta.data_type() == IndexMeta::DataType::DT_FP32) {
|
||||
build_meta_.set_dimension(meta.dimension() - 1);
|
||||
} else {
|
||||
build_meta_.set_dimension(meta.dimension() - 2);
|
||||
}
|
||||
}
|
||||
|
||||
metric_ = IndexFactory::CreateMetric(build_meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("CreateMetric failed, name: %s",
|
||||
build_meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
int ret = metric_->init(build_meta_, build_meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMeasure init failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
raw_meta_.set_builder("DiskAnnBuilder", DiskAnnEntity::kRevision, params);
|
||||
|
||||
ret = entity_.init(meta, max_degree_, list_size_, memory_limit_,
|
||||
build_thread_count_);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
algo_ =
|
||||
DiskAnnAlgorithm::UPointer(new DiskAnnAlgorithm(entity_, max_degree_));
|
||||
|
||||
trainer_ =
|
||||
DiskAnnPqTrainer::UPointer(new DiskAnnPqTrainer(max_train_sample_count_));
|
||||
|
||||
state_ = BUILD_STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::cleanup(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::calculate_entry_point() {
|
||||
size_t dimension = build_meta_.dimension();
|
||||
|
||||
if (build_meta_.data_type() != IndexMeta::DataType::DT_FP32 &&
|
||||
build_meta_.data_type() != IndexMeta::DataType::DT_FP16) {
|
||||
LOG_ERROR("Data type not supported");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::vector<float> centroid_fp32;
|
||||
std::vector<ailego::Float16> centroid_fp16;
|
||||
|
||||
switch (build_meta_.data_type()) {
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
centroid_fp32.resize(dimension);
|
||||
NumericalVectorMean<float> accumulator(dimension);
|
||||
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
|
||||
accumulator.plus(entity_.get_vector(id), dimension * sizeof(float));
|
||||
}
|
||||
accumulator.mean(centroid_fp32.data(), dimension * sizeof(float));
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
centroid_fp16.resize(dimension);
|
||||
NumericalVectorMean<ailego::Float16> accumulator(dimension);
|
||||
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
|
||||
accumulator.plus(entity_.get_vector(id),
|
||||
dimension * sizeof(ailego::Float16));
|
||||
}
|
||||
accumulator.mean(centroid_fp16.data(),
|
||||
dimension * sizeof(ailego::Float16));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
// compute all to one distance
|
||||
diskann_id_t medoid_id = kInvalidId;
|
||||
float min_dist = std::numeric_limits<float>::max();
|
||||
|
||||
switch (build_meta_.data_type()) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
|
||||
const float *data_ptr =
|
||||
reinterpret_cast<const float *>(entity_.get_vector(id));
|
||||
|
||||
float dist = 0.0f;
|
||||
ailego::SquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
centroid_fp32.data(), data_ptr, dimension, &dist);
|
||||
|
||||
if (dist < min_dist) {
|
||||
min_dist = dist;
|
||||
medoid_id = id;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
|
||||
const ailego::Float16 *data_ptr =
|
||||
reinterpret_cast<const ailego::Float16 *>(entity_.get_vector(id));
|
||||
|
||||
float dist = 0.0f;
|
||||
ailego::SquaredEuclideanDistanceMatrix<ailego::Float16, 1, 1>::Compute(
|
||||
centroid_fp16.data(), data_ptr, dimension, &dist);
|
||||
|
||||
if (dist < min_dist) {
|
||||
min_dist = dist;
|
||||
medoid_id = id;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
(*entity_.mutable_medoid()) = medoid_id;
|
||||
|
||||
LOG_INFO("Medoid Calculation Done. ID: %zu", (size_t)medoid_id);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::calculate_pq_chunk_num() {
|
||||
size_t doc_cnt = holder_->count();
|
||||
if (doc_cnt == 0) {
|
||||
LOG_ERROR("Invalid Input. Empty Vecs.");
|
||||
|
||||
return IndexError_InvalidLength;
|
||||
}
|
||||
|
||||
if (memory_limit_set_) {
|
||||
size_t memory_limit_bytes = get_memory_in_bytes(memory_limit_);
|
||||
size_t pq_chunk_num = std::floor(memory_limit_bytes / doc_cnt);
|
||||
if (pq_chunk_num <= 0) {
|
||||
LOG_ERROR("Insufficient memory limit for vec, memory: %zu, vec num: %zu",
|
||||
memory_limit_bytes, doc_cnt);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
}
|
||||
|
||||
pq_chunk_num_ =
|
||||
pq_chunk_num_ < max_pq_chunk_num_ ? pq_chunk_num_ : max_pq_chunk_num_;
|
||||
|
||||
// A chunk num of 0 (public API default) or the internal sentinel means
|
||||
// "auto": quantize into half the dimensions. Resolve it before the
|
||||
// upper-bound check so the default never reaches the divide-by-chunk path.
|
||||
if (pq_chunk_num_ == 0 || pq_chunk_num_ == kDefaultPqChunkNum) {
|
||||
pq_chunk_num_ = build_meta_.dimension() / 2;
|
||||
LOG_INFO(
|
||||
"No Chunk Num input. Quantizing %u dimension data into %u dimension.",
|
||||
build_meta_.dimension(), pq_chunk_num_);
|
||||
}
|
||||
|
||||
if (pq_chunk_num_ > build_meta_.dimension()) {
|
||||
LOG_ERROR("PQ Chunk Num is more than dimension, chunk num: %u, dim: %u",
|
||||
pq_chunk_num_, build_meta_.dimension());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
LOG_INFO("Quantizing %u dimension data into %u bytes.",
|
||||
build_meta_.dimension(), pq_chunk_num_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::build_internal(IndexThreads::Pointer threads) {
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
std::atomic<uint64_t> finished{0};
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(ailego::Closure ::New(this, &DiskAnnBuilder::do_build, i,
|
||||
threads->count(), &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to build index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Built cnt %zu, finished percent %.3f%%", (size_t)finished.load(),
|
||||
finished.load() * 100.0f / entity_.doc_cnt());
|
||||
}
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to build index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::prune_internal(IndexThreads::Pointer threads) {
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
std::atomic<uint64_t> finished{0};
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(ailego::Closure ::New(this, &DiskAnnBuilder::do_prune, i,
|
||||
threads->count(), &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to purne index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Prune cnt %zu, finished percent %.3f%%", (size_t)finished.load(),
|
||||
finished.load() * 100.0f / entity_.doc_cnt());
|
||||
}
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to prune index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::train_quantized_data(IndexThreads::Pointer threads) {
|
||||
LOG_INFO("Starting Train: Chunk Num: %u", pq_chunk_num_);
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
int ret = trainer_->train_quantized_data(
|
||||
threads, holder_, build_meta_, entity_.pq_full_pivot_data(),
|
||||
entity_.pq_centroid(), entity_.pq_chunk_offsets(), pq_chunk_num_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Train Quantized Data Error, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t pq_time = timer.milli_seconds();
|
||||
LOG_INFO("Train Quantized Data Done, time: %zu ms", pq_time);
|
||||
|
||||
(*entity_.mutable_pq_meta()).full_pivot_data_size =
|
||||
entity_.pq_full_pivot_data().size();
|
||||
(*entity_.mutable_pq_meta()).centroid_data_size =
|
||||
entity_.pq_centroid().size();
|
||||
(*entity_.mutable_pq_meta()).chunk_num = pq_chunk_num_;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::generate_quantized_data(IndexThreads::Pointer threads) {
|
||||
LOG_INFO("Starting PQ Generate: Query Memory Limit: %lf, Chunk Num: %u",
|
||||
memory_limit_, pq_chunk_num_);
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
int ret = trainer_->generate_quantized_data(
|
||||
threads, holder_, build_meta_, entity_.pq_centroid(),
|
||||
entity_.block_compressed_data(), pq_chunk_num_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Generate Quantized Data Error, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t pq_time = timer.milli_seconds();
|
||||
LOG_INFO("Generate Quantized Data Done, time: %zu ms", pq_time);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void DiskAnnBuilder::do_build(uint64_t idx, size_t step_size,
|
||||
std::atomic<uint64_t> *finished) {
|
||||
AILEGO_DEFER([&]() {
|
||||
std::lock_guard<std::mutex> latch(mutex_);
|
||||
cond_.notify_one();
|
||||
});
|
||||
|
||||
DiskAnnContext *ctx = new (std::nothrow) DiskAnnContext(
|
||||
build_meta_, metric_,
|
||||
std::shared_ptr<DiskAnnEntity>(&entity_, [](DiskAnnEntity *) {}));
|
||||
|
||||
if (ailego_unlikely(ctx == nullptr)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("Failed to create context");
|
||||
errcode_ = IndexError_NoMemory;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
DiskAnnContext::Pointer auto_ptr(ctx);
|
||||
ctx->init(DiskAnnContext::kBuilderContext, max_degree_, pq_chunk_num_,
|
||||
build_meta_.element_size());
|
||||
ctx->set_list_size(list_size_);
|
||||
|
||||
for (uint64_t id = idx; id < entity_.doc_cnt(); id += step_size) {
|
||||
ctx->reset_query(entity_.get_vector(id));
|
||||
int ret = algo_->add_node(id, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("DiskAnn graph add node failed");
|
||||
errcode_ = ret;
|
||||
}
|
||||
return;
|
||||
}
|
||||
ctx->clear();
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
void DiskAnnBuilder::do_prune(uint64_t idx, size_t step_size,
|
||||
std::atomic<uint64_t> *finished) {
|
||||
AILEGO_DEFER([&]() {
|
||||
std::lock_guard<std::mutex> latch(mutex_);
|
||||
cond_.notify_one();
|
||||
});
|
||||
|
||||
DiskAnnContext *ctx = new (std::nothrow) DiskAnnContext(
|
||||
build_meta_, metric_,
|
||||
std::shared_ptr<DiskAnnEntity>(&entity_, [](DiskAnnEntity *) {}));
|
||||
|
||||
if (ailego_unlikely(ctx == nullptr)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("Failed to create context");
|
||||
errcode_ = IndexError_NoMemory;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
DiskAnnContext::Pointer auto_ptr(ctx);
|
||||
ctx->init(DiskAnnContext::kBuilderContext, max_degree_, pq_chunk_num_,
|
||||
build_meta_.element_size());
|
||||
ctx->set_list_size(list_size_);
|
||||
|
||||
for (uint64_t id = idx; id < entity_.doc_cnt(); id += step_size) {
|
||||
ctx->reset_query(entity_.get_vector(id));
|
||||
int ret = algo_->prune_node(id, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("DiskAnn graph add node failed");
|
||||
errcode_ = ret;
|
||||
}
|
||||
return;
|
||||
}
|
||||
ctx->clear();
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::train(const IndexTrainer::Pointer & /*trainer*/) {
|
||||
if (state_ != BUILD_STATE_INITED) {
|
||||
LOG_ERROR("Init the builder before DiskAnnBuilder::train");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Begin DiskAnnBuilder::train by trainer");
|
||||
|
||||
stats_.set_trained_count(0UL);
|
||||
stats_.set_trained_costtime(0UL);
|
||||
state_ = BUILD_STATE_TRAINED;
|
||||
|
||||
LOG_INFO("End DiskAnnBuilder::train by trainer");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::train(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) {
|
||||
if (state_ != BUILD_STATE_INITED) {
|
||||
LOG_ERROR("Init the builder before DiskAnnBuilder::train");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Begin DiskAnnBuilder::train");
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
holder_ = std::move(holder);
|
||||
|
||||
LOG_INFO("Start to calculate chunk num");
|
||||
int ret = calculate_pq_chunk_num();
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(build_thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
|
||||
ret = train_quantized_data(threads);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
stats_.set_trained_count(holder_->count());
|
||||
|
||||
stats_.set_trained_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
|
||||
state_ = BUILD_STATE_TRAINED;
|
||||
|
||||
holder_.reset();
|
||||
|
||||
LOG_INFO("End DiskAnnBuilder::train");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::do_norm(const void *data_ptr, std::string *norm_data) {
|
||||
size_t dimension = build_meta_.dimension();
|
||||
const float *float_data_ptr = reinterpret_cast<const float *>(data_ptr);
|
||||
|
||||
norm_data->resize(dimension * sizeof(float));
|
||||
float *output_buf = reinterpret_cast<float *>(&((*norm_data)[0]));
|
||||
std::memcpy(output_buf, float_data_ptr, dimension * sizeof(float));
|
||||
|
||||
float norm = 0.0f;
|
||||
ailego::Normalizer<float>::L2(output_buf, dimension, &norm);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) {
|
||||
LOG_INFO("Start DiskAnnBuilder::build");
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
holder_ = holder;
|
||||
|
||||
if (!threads) {
|
||||
threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(build_thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(holder->count() == 0)) {
|
||||
LOG_ERROR("Holder is empty");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
int ret = entity_.reserve_space(holder->count());
|
||||
|
||||
error_ = false;
|
||||
while (iter->is_valid()) {
|
||||
ret = entity_.add_vector(iter->key(), iter->data());
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
iter->next();
|
||||
}
|
||||
|
||||
LOG_INFO("Finished saving vector");
|
||||
|
||||
LOG_INFO("Start to calculate entrypoint");
|
||||
ret = calculate_entry_point();
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("Start to build vamana graph");
|
||||
ret = build_internal(threads);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("Start final cleanup..");
|
||||
ret = prune_internal(threads);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("Start to generate quantized data");
|
||||
ret = generate_quantized_data(threads);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
state_ = BUILD_STATE_BUILT;
|
||||
|
||||
stats_.set_built_count(entity_.doc_cnt());
|
||||
stats_.set_built_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
|
||||
LOG_INFO("End DiskAnnBuilder::build");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilder::dump(const IndexDumper::Pointer &dumper) {
|
||||
if (state_ != BUILD_STATE_BUILT) {
|
||||
LOG_INFO("Build the index before DiskAnnBuilder::dump");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Begin DiskAnnBuilder::dump");
|
||||
|
||||
raw_meta_.set_searcher("DiskAnnSearcher", 0, ailego::Params());
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
int ret = IndexHelper::SerializeToDumper(raw_meta_, dumper.get());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to serialize meta into dumper.");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = entity_.dump(holder_, raw_meta_, dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Index dump failed, ret: %u", ret);
|
||||
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
stats_.set_dumped_count(holder_->count());
|
||||
stats_.set_dumped_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
|
||||
LOG_INFO("DiskAnnBuilder::dump");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_BUILDER(DiskAnnBuilder);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,131 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_builder.h>
|
||||
#include "diskann_algorithm.h"
|
||||
#include "diskann_builder_entity.h"
|
||||
#include "diskann_pq_trainer.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnBuilder : public IndexBuilder {
|
||||
public:
|
||||
//! Constructor
|
||||
DiskAnnBuilder() = default;
|
||||
|
||||
//! Initialize the builder
|
||||
virtual int init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup the builder
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(const IndexTrainer::Pointer &trainer) override;
|
||||
|
||||
//! Build the index
|
||||
virtual int build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Dump index into storage
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
int do_norm(const void *data_ptr, std::string *norm_data);
|
||||
|
||||
private:
|
||||
int train_quantized_data(IndexThreads::Pointer threads);
|
||||
int generate_quantized_data(IndexThreads::Pointer threads);
|
||||
int build_internal(IndexThreads::Pointer threads);
|
||||
int prune_internal(IndexThreads::Pointer threads);
|
||||
|
||||
void do_build(uint64_t idx, size_t step_size,
|
||||
std::atomic<uint64_t> *finished);
|
||||
|
||||
void do_prune(uint64_t idx, size_t step_size,
|
||||
std::atomic<uint64_t> *finished);
|
||||
|
||||
int calculate_entry_point();
|
||||
|
||||
int calculate_pq_chunk_num();
|
||||
|
||||
double get_memory_in_bytes(double search_ram_budget) {
|
||||
return search_ram_budget * 1024 * 1024 * 1024;
|
||||
}
|
||||
|
||||
private:
|
||||
enum BUILD_STATE {
|
||||
BUILD_STATE_INIT = 0,
|
||||
BUILD_STATE_INITED = 1,
|
||||
BUILD_STATE_TRAINED = 2,
|
||||
BUILD_STATE_BUILT = 3
|
||||
};
|
||||
|
||||
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
|
||||
constexpr static uint32_t kDefaultListSize = 50U;
|
||||
constexpr static uint32_t kDefaultMaxDegree = 100U;
|
||||
constexpr static uint32_t kDefaultPqChunkNum = -1U;
|
||||
|
||||
std::string data_file_;
|
||||
|
||||
uint32_t max_degree_{kDefaultMaxDegree};
|
||||
uint32_t list_size_{kDefaultListSize};
|
||||
double memory_limit_{0.0};
|
||||
bool memory_limit_set_{false};
|
||||
uint32_t max_pq_chunk_num_{kDefaultPqChunkNum};
|
||||
uint32_t pq_chunk_num_{kDefaultPqChunkNum};
|
||||
uint32_t build_thread_count_{0};
|
||||
uint32_t max_train_sample_count_{PQTable::kMaxTrainSampleCount};
|
||||
double train_sample_ratio_{PQTable::kTrainSampleRatio};
|
||||
std::string universal_label_{""};
|
||||
std::string codebook_prefix_{""};
|
||||
std::string index_path_prefix_{"./diskann"};
|
||||
|
||||
BUILD_STATE state_;
|
||||
Stats stats_;
|
||||
|
||||
int errcode_{0};
|
||||
std::atomic_bool error_{false};
|
||||
|
||||
IndexMetric::Pointer metric_{};
|
||||
|
||||
std::mutex mutex_{};
|
||||
std::condition_variable cond_{};
|
||||
|
||||
IndexMeta raw_meta_;
|
||||
IndexMeta build_meta_;
|
||||
|
||||
DiskAnnBuilderEntity entity_{};
|
||||
|
||||
IndexHolder::Pointer holder_;
|
||||
|
||||
DiskAnnAlgorithm::UPointer algo_;
|
||||
DiskAnnPqTrainer::UPointer trainer_;
|
||||
|
||||
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,643 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_builder_entity.h"
|
||||
#include <iostream>
|
||||
#include "diskann_algorithm.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int DiskAnnBuilderEntity::init(const IndexMeta &meta, uint32_t max_degree,
|
||||
uint32_t list_size, double memory_limit,
|
||||
uint32_t build_threads) {
|
||||
meta_ = meta;
|
||||
|
||||
max_degree_ = max_degree;
|
||||
list_size_ = list_size;
|
||||
|
||||
memory_limit_ = memory_limit;
|
||||
|
||||
num_threads_ = build_threads;
|
||||
|
||||
max_build_degree_ = max_degree_ * kDefaultGraphSlackFactor;
|
||||
|
||||
neighbor_size_ = sizeof(uint32_t) + max_build_degree_ * sizeof(diskann_id_t);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::reserve_space(uint32_t docs) {
|
||||
vectors_buffer_.reserve(meta_.element_size() * docs);
|
||||
keys_buffer_.reserve(sizeof(diskann_key_t) * docs);
|
||||
neighbors_buffer_.reserve(neighbor_size_ * docs);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::add_vector(diskann_key_t key, const void *vec) {
|
||||
vectors_buffer_.append(reinterpret_cast<const char *>(vec),
|
||||
meta_.element_size());
|
||||
keys_buffer_.append(reinterpret_cast<const char *>(&key), sizeof(key));
|
||||
|
||||
uint32_t neighbor_cnt = 0;
|
||||
std::vector<diskann_id_t> neighbor{max_build_degree_, 0};
|
||||
|
||||
neighbors_buffer_.append(reinterpret_cast<const char *>(&neighbor_cnt),
|
||||
sizeof(uint32_t));
|
||||
neighbors_buffer_.append(reinterpret_cast<const char *>(neighbor.data()),
|
||||
sizeof(diskann_id_t) * max_build_degree_);
|
||||
|
||||
(*mutable_doc_cnt())++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const void *DiskAnnBuilderEntity::get_vector(diskann_id_t id) const {
|
||||
size_t offset = (size_t)id * meta_.element_size();
|
||||
return vectors_buffer_.data() + offset;
|
||||
}
|
||||
|
||||
diskann_key_t DiskAnnBuilderEntity::get_key(diskann_id_t id) const {
|
||||
size_t offset = (size_t)id * sizeof(diskann_key_t);
|
||||
|
||||
return *(
|
||||
reinterpret_cast<const diskann_key_t *>(keys_buffer_.data() + offset));
|
||||
}
|
||||
|
||||
//! Get vector local id by key
|
||||
diskann_id_t DiskAnnBuilderEntity::get_id(diskann_key_t /*key*/) const {
|
||||
LOG_ERROR("DiskAnnBuilderEntity::get_id not implemented.");
|
||||
return kInvalidId;
|
||||
}
|
||||
|
||||
std::pair<uint32_t, const diskann_id_t *> DiskAnnBuilderEntity::get_neighbors(
|
||||
diskann_id_t id) const {
|
||||
size_t offset = (size_t)id * neighbor_size_;
|
||||
|
||||
const uint8_t *start_ptr =
|
||||
reinterpret_cast<const uint8_t *>(neighbors_buffer_.data()) + offset;
|
||||
|
||||
uint32_t neighbor_cnt = *(reinterpret_cast<const uint32_t *>(start_ptr));
|
||||
|
||||
const diskann_id_t *neighbors =
|
||||
reinterpret_cast<const diskann_id_t *>(start_ptr + sizeof(uint32_t));
|
||||
|
||||
return std::make_pair(neighbor_cnt, neighbors);
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::set_neighbors(
|
||||
diskann_id_t id, const std::vector<diskann_id_t> &neighbor_ids) {
|
||||
size_t offset = (size_t)id * neighbor_size_;
|
||||
|
||||
uint8_t *start_ptr =
|
||||
reinterpret_cast<uint8_t *>(&neighbors_buffer_[0]) + offset;
|
||||
|
||||
uint32_t neighbor_cnt = neighbor_ids.size();
|
||||
|
||||
memcpy(start_ptr + sizeof(uint32_t), neighbor_ids.data(),
|
||||
sizeof(diskann_id_t) * neighbor_cnt);
|
||||
memcpy(start_ptr, &neighbor_cnt, sizeof(uint32_t));
|
||||
|
||||
if (max_observed_degree_ < neighbor_cnt) {
|
||||
max_observed_degree_ = neighbor_cnt;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::add_neighbor(diskann_id_t id,
|
||||
diskann_id_t neighbor_id) {
|
||||
size_t offset = (size_t)id * neighbor_size_;
|
||||
|
||||
uint8_t *start_ptr =
|
||||
reinterpret_cast<uint8_t *>(&neighbors_buffer_[0]) + offset;
|
||||
|
||||
uint32_t neighbor_cnt = *reinterpret_cast<uint32_t *>(start_ptr);
|
||||
|
||||
memcpy(start_ptr + sizeof(uint32_t) + sizeof(diskann_id_t) * neighbor_cnt,
|
||||
&neighbor_id, sizeof(diskann_id_t));
|
||||
|
||||
neighbor_cnt += 1;
|
||||
|
||||
memcpy(start_ptr, &neighbor_cnt, sizeof(uint32_t));
|
||||
|
||||
if (max_observed_degree_ < neighbor_cnt) {
|
||||
max_observed_degree_ = neighbor_cnt;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int64_t DiskAnnBuilderEntity::dump_segment(const IndexDumper::Pointer &dumper,
|
||||
const std::string &segment_id,
|
||||
const void *data,
|
||||
size_t size) const {
|
||||
size_t len = dumper->write(data, size);
|
||||
if (len != size) {
|
||||
LOG_ERROR("Dump segment %s data failed, expect: %lu, actual: %lu",
|
||||
segment_id.c_str(), size, len);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
size_t padding_size = AlignSize(size) - size;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t crc = ailego::Crc32c::Hash(data, size);
|
||||
int ret = dumper->append(segment_id, size, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_pq_meta_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
uint32_t crc = 0U;
|
||||
|
||||
// write meta
|
||||
size_t size_pq_meta = dumper->write(&pq_meta_, sizeof(DiskAnnPqMeta));
|
||||
if (size_pq_meta != sizeof(DiskAnnPqMeta)) {
|
||||
LOG_ERROR("Failed to dump PQ meta data, expect: %lu, actual: %lu",
|
||||
sizeof(DiskAnnPqMeta), size_pq_meta);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(&pq_meta_, sizeof(DiskAnnPqMeta), crc);
|
||||
|
||||
// write full pivot data
|
||||
size_t size_full_pivot_data =
|
||||
dumper->write(pq_full_pivot_data_.data(), pq_meta_.full_pivot_data_size);
|
||||
if (size_full_pivot_data != pq_meta_.full_pivot_data_size) {
|
||||
LOG_ERROR("Failed to dump full pivot data, expect: %zu, actual: %zu",
|
||||
(size_t)pq_meta_.full_pivot_data_size, size_full_pivot_data);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(pq_full_pivot_data_.data(),
|
||||
pq_meta_.full_pivot_data_size, crc);
|
||||
|
||||
// write centroid num
|
||||
size_t size_centroid =
|
||||
dumper->write(pq_centroid_.data(), pq_meta_.centroid_data_size);
|
||||
if (size_centroid != pq_meta_.centroid_data_size) {
|
||||
LOG_ERROR("Failed to dump centroid num, expect: %zu, actual: %zu",
|
||||
(size_t)pq_meta_.centroid_data_size, size_centroid);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(pq_centroid_.data(), pq_meta_.centroid_data_size,
|
||||
crc);
|
||||
|
||||
// write chunk offset
|
||||
size_t size_chunk_offset = dumper->write(
|
||||
pq_chunk_offsets_.data(), (pq_meta_.chunk_num + 1) * sizeof(uint32_t));
|
||||
if (size_chunk_offset != (pq_meta_.chunk_num + 1) * sizeof(uint32_t)) {
|
||||
LOG_ERROR("Failed to dump centroid num, expect: %zu, actual: %zu",
|
||||
(size_t)((pq_meta_.chunk_num + 1) * sizeof(uint32_t)),
|
||||
size_chunk_offset);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(pq_chunk_offsets_.data(),
|
||||
(pq_meta_.chunk_num + 1) * sizeof(uint32_t), crc);
|
||||
|
||||
// write size
|
||||
size_t size_total =
|
||||
size_pq_meta + size_full_pivot_data + size_centroid + size_chunk_offset;
|
||||
|
||||
// write pad
|
||||
size_t padding_size = AlignSize(size_total) - size_total;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
int ret =
|
||||
dumper->append(kDiskAnnPqMetaSegmentId, size_total, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump PQ segment failed, ret %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_pq_data_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
uint64_t doc_cnt = meta_header_.doc_cnt;
|
||||
uint64_t chunk_num = pq_meta_.chunk_num;
|
||||
|
||||
uint32_t crc = 0U;
|
||||
|
||||
// write pq data
|
||||
size_t size_total =
|
||||
dumper->write(block_compressed_data_.data(), doc_cnt * chunk_num);
|
||||
|
||||
if (size_total != doc_cnt * chunk_num) {
|
||||
LOG_ERROR("Failed to dump block compressed data, expect: %zu, actual: %zu",
|
||||
(size_t)(doc_cnt * chunk_num), size_total);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(block_compressed_data_.data(), doc_cnt * chunk_num,
|
||||
crc);
|
||||
|
||||
// write pad
|
||||
size_t padding_size = AlignSize(size_total) - size_total;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
int ret =
|
||||
dumper->append(kDiskAnnPqDataSegmentId, size_total, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump PQ data segment failed, ret %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_dummy_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
// to make offset aligned with 4K
|
||||
size_t dumper_header_size = dumper->size();
|
||||
|
||||
size_t dummy_size =
|
||||
DiskAnnUtil::round_up(dumper_header_size, DiskAnnUtil::kSectorSize) -
|
||||
dumper_header_size;
|
||||
|
||||
if (dummy_size != 0) {
|
||||
std::string dummpy_data(dummy_size, '\0');
|
||||
if (dumper->write(dummpy_data.data(), dummy_size) != dummy_size) {
|
||||
LOG_ERROR("write dummy failed, size %lu", dummy_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
int ret = dumper->append(kDiskAnnDummpySegmentId, dummy_size, 0, 0);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump dummy data segment failed, ret %d", ret);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_key_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
//! Dump keys
|
||||
size_t key_segment_size = doc_cnt() * sizeof(diskann_key_t);
|
||||
int64_t keys_size = dump_segment(dumper, kDiskAnnKeySegmentId,
|
||||
keys_buffer_.data(), key_segment_size);
|
||||
if (keys_size < 0) {
|
||||
return keys_size;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_key_mapping_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
std::vector<diskann_id_t> mapping(doc_cnt());
|
||||
|
||||
const diskann_key_t *keys = reinterpret_cast<diskann_key_t *>(
|
||||
const_cast<char *>(keys_buffer_.data()));
|
||||
|
||||
std::iota(mapping.begin(), mapping.end(), 0U);
|
||||
std::sort(mapping.begin(), mapping.end(),
|
||||
[&](diskann_id_t i, diskann_id_t j) { return keys[i] < keys[j]; });
|
||||
|
||||
size_t size = mapping.size() * sizeof(diskann_id_t);
|
||||
int64_t ret =
|
||||
dump_segment(dumper, kDiskAnnKeyMappingSegmentId, mapping.data(), size);
|
||||
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump vectors segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump_entrypoint_segment(
|
||||
const IndexDumper::Pointer &dumper) const {
|
||||
std::string entrypoint_buffer;
|
||||
|
||||
size_t size = sizeof(uint32_t) + entrypoints_.size() * sizeof(diskann_id_t);
|
||||
entrypoint_buffer.resize(size);
|
||||
|
||||
uint8_t *buffer_ptr = reinterpret_cast<uint8_t *>(&entrypoint_buffer[0]);
|
||||
|
||||
uint32_t point_cnt = entrypoints_.size();
|
||||
memcpy(buffer_ptr, &point_cnt, sizeof(uint32_t));
|
||||
memcpy(buffer_ptr + sizeof(uint32_t), entrypoints_.data(),
|
||||
entrypoints_.size() * sizeof(diskann_id_t));
|
||||
|
||||
int64_t ret = dump_segment(dumper, kDiskAnnEntryPointSegmentId,
|
||||
entrypoint_buffer.data(), size);
|
||||
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump entrypoint segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnBuilderEntity::dump(IndexHolder::Pointer holder, IndexMeta &meta,
|
||||
const IndexDumper::Pointer &dumper) {
|
||||
uint64_t doc_cnt = holder->count();
|
||||
uint64_t max_node_size =
|
||||
(uint64_t)max_observed_degree_ * sizeof(diskann_id_t) + sizeof(uint32_t) +
|
||||
meta_.element_size();
|
||||
uint64_t node_per_sector =
|
||||
DiskAnnUtil::kSectorSize /
|
||||
max_node_size; // 0 if max_node_size > DiskAnnUtil::kSectorSize
|
||||
|
||||
std::string node_buf;
|
||||
node_buf.resize(max_node_size);
|
||||
|
||||
diskann_id_t *neighbor_buf =
|
||||
(diskann_id_t *)(node_buf.data() + (meta_.element_size()) +
|
||||
sizeof(uint32_t));
|
||||
|
||||
LOG_INFO(
|
||||
"Dump Data, medoid: %zu, max node size: %zu, node per sector: %zu, "
|
||||
"max observed degree: %zu",
|
||||
(size_t)medoid(), (size_t)max_node_size, (size_t)node_per_sector,
|
||||
(size_t)max_observed_degree_);
|
||||
|
||||
// write a dummy segment to make data align
|
||||
int ret = dump_dummy_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump dummy segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump data by sector
|
||||
size_t write_size = 0;
|
||||
uint32_t crc = 0U;
|
||||
size_t len = 0;
|
||||
|
||||
// no need to write first sector
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
uint64_t index_size = 0;
|
||||
uint32_t neighbor_num;
|
||||
if (node_per_sector > 0) {
|
||||
uint64_t sector_num =
|
||||
DiskAnnUtil::round_up(doc_cnt, node_per_sector) / node_per_sector;
|
||||
|
||||
diskann_id_t cur_node_id = 0;
|
||||
|
||||
std::string sector_buf;
|
||||
sector_buf.resize(DiskAnnUtil::kSectorSize);
|
||||
|
||||
for (uint64_t sector = 0; sector < sector_num; sector++) {
|
||||
if (sector != 0 && sector % 100000 == 0) {
|
||||
LOG_INFO("Sector #%zu written", (size_t)sector);
|
||||
}
|
||||
|
||||
memset(&(sector_buf[0]), 0, DiskAnnUtil::kSectorSize);
|
||||
|
||||
for (uint64_t sector_node_id = 0;
|
||||
sector_node_id < node_per_sector && cur_node_id < doc_cnt;
|
||||
sector_node_id++) {
|
||||
memset(&(node_buf[0]), 0, max_node_size);
|
||||
|
||||
auto neighbors = get_neighbors(cur_node_id);
|
||||
neighbor_num = neighbors.first;
|
||||
|
||||
ailego_assert(neighbor_num > 0);
|
||||
ailego_assert(neighbor_num <= max_observed_degree_);
|
||||
|
||||
memcpy(&(neighbor_buf[0]), neighbors.second,
|
||||
neighbors.first * sizeof(diskann_id_t));
|
||||
|
||||
if (iter->is_valid()) {
|
||||
const void *vec = iter->data();
|
||||
memcpy(&(node_buf[0]), vec, meta.element_size());
|
||||
|
||||
iter->next();
|
||||
} else {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
// write neighbor num
|
||||
*(uint32_t *)(node_buf.data() + meta_.element_size()) = neighbor_num;
|
||||
|
||||
// write neighbor buffer
|
||||
memcpy(&(node_buf[0]) + meta_.element_size() + sizeof(uint32_t),
|
||||
neighbor_buf, neighbor_num * sizeof(diskann_id_t));
|
||||
|
||||
// get offset into sector_buf
|
||||
char *sector_node_buf = §or_buf[sector_node_id * max_node_size];
|
||||
|
||||
// copy node buf into sector_node_buf
|
||||
memcpy(sector_node_buf, node_buf.data(), max_node_size);
|
||||
|
||||
cur_node_id++;
|
||||
}
|
||||
|
||||
// flush sector to disk
|
||||
len = dumper->write(sector_buf.data(), DiskAnnUtil::kSectorSize);
|
||||
if (len != DiskAnnUtil::kSectorSize) {
|
||||
LOG_ERROR("Write Vector Error, write=%zu, expect=%zu", len,
|
||||
(size_t)DiskAnnUtil::kSectorSize);
|
||||
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
write_size += DiskAnnUtil::kSectorSize;
|
||||
crc = ailego::Crc32c::Hash(sector_buf.data(), DiskAnnUtil::kSectorSize,
|
||||
crc);
|
||||
}
|
||||
|
||||
LOG_INFO("Total Sector #%zu written", (size_t)sector_num);
|
||||
|
||||
index_size = sector_num * DiskAnnUtil::kSectorSize;
|
||||
} else {
|
||||
// Write multi-sector nodes
|
||||
std::string multisector_buf;
|
||||
multisector_buf.resize(
|
||||
DiskAnnUtil::round_up(max_node_size, DiskAnnUtil::kSectorSize));
|
||||
|
||||
uint64_t sector_num_per_node =
|
||||
DiskAnnUtil::div_round_up(max_node_size, DiskAnnUtil::kSectorSize);
|
||||
|
||||
for (uint64_t i = 0; i < doc_cnt; i++) {
|
||||
if (i != 0 && (i * sector_num_per_node) % 100000 == 0) {
|
||||
LOG_INFO("Sector # %zu written", (size_t)(i * sector_num_per_node));
|
||||
}
|
||||
|
||||
memset(&(multisector_buf[0]), 0,
|
||||
sector_num_per_node * DiskAnnUtil::kSectorSize);
|
||||
memset(&(node_buf[0]), 0, max_node_size);
|
||||
|
||||
auto neighbors = get_neighbors(i);
|
||||
neighbor_num = neighbors.first;
|
||||
|
||||
ailego_assert(neighbor_num > 0);
|
||||
ailego_assert(neighbor_num <= max_observed_degree_);
|
||||
|
||||
// read node's nhood
|
||||
memcpy((char *)neighbor_buf, neighbors.second,
|
||||
neighbor_num * sizeof(diskann_id_t));
|
||||
|
||||
if (iter->is_valid()) {
|
||||
const void *vec = iter->data();
|
||||
memcpy(&(multisector_buf[0]), vec, meta.element_size());
|
||||
|
||||
iter->next();
|
||||
} else {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
// write neighbor
|
||||
*(uint32_t *)(&(multisector_buf[0]) + meta_.element_size()) =
|
||||
neighbor_num;
|
||||
|
||||
// write nhood next
|
||||
memcpy(&(multisector_buf[0]) + meta_.element_size() + sizeof(uint32_t),
|
||||
neighbor_buf, neighbor_num * sizeof(diskann_id_t));
|
||||
|
||||
// flush sector to disk
|
||||
len = dumper->write(multisector_buf.data(),
|
||||
sector_num_per_node * DiskAnnUtil::kSectorSize);
|
||||
if (len != sector_num_per_node * DiskAnnUtil::kSectorSize) {
|
||||
LOG_ERROR("Write Vector Error, write=%zu, expect=%zu", len,
|
||||
(size_t)(sector_num_per_node * DiskAnnUtil::kSectorSize));
|
||||
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
write_size += sector_num_per_node * DiskAnnUtil::kSectorSize;
|
||||
|
||||
crc = ailego::Crc32c::Hash(multisector_buf.data(),
|
||||
sector_num_per_node * DiskAnnUtil::kSectorSize,
|
||||
crc);
|
||||
}
|
||||
|
||||
LOG_INFO("Total Sector #%zu written",
|
||||
(size_t)(doc_cnt * sector_num_per_node));
|
||||
|
||||
index_size = doc_cnt * sector_num_per_node * DiskAnnUtil::kSectorSize;
|
||||
}
|
||||
|
||||
size_t padding_size = AlignSize(write_size) - write_size;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
ret = dumper->append(kDiskAnnVectorSegmentId, write_size, 0UL, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump vectors segment failed, ret %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump diskann meta
|
||||
meta_header_.doc_cnt = doc_cnt;
|
||||
meta_header_.ndims = meta_.dimension();
|
||||
meta_header_.medoid = medoid();
|
||||
meta_header_.max_node_size = max_node_size;
|
||||
meta_header_.max_degree = max_observed_degree_;
|
||||
meta_header_.node_per_sector = node_per_sector;
|
||||
meta_header_.vamana_frozen_num = 0;
|
||||
meta_header_.vamana_frozen_loc = medoid();
|
||||
meta_header_.append_reorder_data = 0;
|
||||
meta_header_.index_size = index_size;
|
||||
|
||||
ret = dump_segment(dumper, kDiskAnnMetaSegmentId, &meta_header_,
|
||||
sizeof(DiskAnnMetaHeader));
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump vectors segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump pq meta
|
||||
ret = dump_pq_meta_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump pq meta segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump pq data
|
||||
ret = dump_pq_data_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump pq data segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump key
|
||||
ret = dump_key_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump key segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump key mapping
|
||||
ret = dump_key_mapping_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump key mapping segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// dump entrypint
|
||||
ret = dump_entrypoint_segment(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump entrypoint segment failed");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("DiskAnn Index File Dumped");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,107 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
// wrapper class aligned with diskann
|
||||
class DiskAnnBuilderEntity : public DiskAnnEntity {
|
||||
public:
|
||||
using Pointer = std::shared_ptr<DiskAnnBuilderEntity>;
|
||||
|
||||
DiskAnnBuilderEntity() = default;
|
||||
virtual ~DiskAnnBuilderEntity() = default;
|
||||
|
||||
public:
|
||||
int add_vector(diskann_key_t key, const void *vec) override;
|
||||
|
||||
std::pair<uint32_t, const diskann_id_t *> get_neighbors(
|
||||
diskann_id_t id) const override;
|
||||
|
||||
int set_neighbors(diskann_id_t id,
|
||||
const std::vector<diskann_id_t> &neighbor_ids) override;
|
||||
|
||||
int add_neighbor(diskann_id_t id, diskann_id_t neighbor_id) override;
|
||||
|
||||
diskann_id_t get_id(diskann_key_t key) const override;
|
||||
diskann_key_t get_key(diskann_id_t id) const override;
|
||||
const void *get_vector(diskann_id_t id) const override;
|
||||
|
||||
public:
|
||||
int init(const IndexMeta &meta, uint32_t max_degree, uint32_t list_size,
|
||||
double memory_limit, uint32_t build_threads);
|
||||
|
||||
int dump(IndexHolder::Pointer holder, IndexMeta &meta,
|
||||
const IndexDumper::Pointer &dumper);
|
||||
|
||||
int64_t dump_segment(const IndexDumper::Pointer &dumper,
|
||||
const std::string &segment_id, const void *data,
|
||||
size_t size) const;
|
||||
int dump_dummy_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_pq_meta_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_pq_data_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_key_mapping_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_entrypoint_segment(const IndexDumper::Pointer &dumper) const;
|
||||
int dump_key_segment(const IndexDumper::Pointer &dumper) const;
|
||||
|
||||
int reserve_space(uint32_t docs);
|
||||
|
||||
std::vector<uint8_t> &pq_full_pivot_data() {
|
||||
return pq_full_pivot_data_;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> &pq_centroid() {
|
||||
return pq_centroid_;
|
||||
}
|
||||
|
||||
std::vector<uint32_t> &pq_chunk_offsets() {
|
||||
return pq_chunk_offsets_;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> &block_compressed_data() {
|
||||
return block_compressed_data_;
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t max_degree_{0};
|
||||
uint32_t list_size_{0};
|
||||
double memory_limit_{0};
|
||||
uint32_t num_threads_{0};
|
||||
uint32_t max_build_degree_{0};
|
||||
uint32_t max_observed_degree_{0};
|
||||
uint32_t neighbor_size_{0};
|
||||
|
||||
std::string mem_index_file_{""};
|
||||
std::string index_path_prefix_{""};
|
||||
|
||||
std::string vectors_buffer_{};
|
||||
std::string keys_buffer_{};
|
||||
std::string neighbors_buffer_{};
|
||||
std::vector<diskann_id_t> entrypoints_{};
|
||||
|
||||
IndexMeta meta_;
|
||||
|
||||
std::vector<uint8_t> pq_full_pivot_data_;
|
||||
std::vector<uint8_t> pq_centroid_;
|
||||
std::vector<uint32_t> pq_chunk_offsets_;
|
||||
std::vector<uint8_t> block_compressed_data_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,134 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_context.h"
|
||||
#include <chrono>
|
||||
#include "diskann_pq_table.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnContext::DiskAnnContext(const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &measure,
|
||||
const DiskAnnEntity::Pointer &entity)
|
||||
: dc_(entity.get(), measure, meta.dimension()), entity_{entity} {}
|
||||
|
||||
int DiskAnnContext::init(ContextType type, uint32_t graph_degree,
|
||||
uint32_t pq_chunk_num, uint32_t element_size) {
|
||||
type_ = type;
|
||||
element_size_ = element_size;
|
||||
pq_chunk_num_ = pq_chunk_num;
|
||||
|
||||
DiskAnnUtil::alloc_aligned((void **)&query_, element_size_, 32);
|
||||
DiskAnnUtil::alloc_aligned((void **)&query_rotated_, element_size_, 32);
|
||||
|
||||
int ret;
|
||||
switch (type) {
|
||||
case kBuilderContext:
|
||||
ret = visit_filter_.init(VisitFilter::ByteMap, entity_->doc_cnt(),
|
||||
entity_->doc_cnt(), negative_probility_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
|
||||
case kSearcherContext:
|
||||
ret = visit_filter_.init(filter_mode_, entity_->doc_cnt(),
|
||||
entity_->doc_cnt(), negative_probility_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
|
||||
DiskAnnUtil::alloc_aligned(
|
||||
(void **)&pq_table_dist_buffer_,
|
||||
PQTable::kPQCentroidNum * pq_chunk_num_ * sizeof(float), 256);
|
||||
DiskAnnUtil::alloc_aligned((void **)&pq_coord_buffer_,
|
||||
graph_degree * pq_chunk_num_ * sizeof(uint8_t),
|
||||
256);
|
||||
DiskAnnUtil::alloc_aligned((void **)&coord_buffer_, element_size_, 256);
|
||||
DiskAnnUtil::alloc_aligned(
|
||||
(void **)§or_buffer_,
|
||||
DiskAnnUtil::kMaxSectorReadNum * DiskAnnUtil::kSectorSize,
|
||||
DiskAnnUtil::kSectorSize);
|
||||
|
||||
ret = setup_io_ctx(io_ctx_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("setup io ctx error, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
LOG_ERROR("Init context failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
DiskAnnContext::~DiskAnnContext() {
|
||||
free(query_);
|
||||
free(query_rotated_);
|
||||
free(pq_table_dist_buffer_);
|
||||
free(pq_coord_buffer_);
|
||||
free(coord_buffer_);
|
||||
free(sector_buffer_);
|
||||
|
||||
if (type_ == kSearcherContext) {
|
||||
destroy_io_ctx(io_ctx_);
|
||||
}
|
||||
}
|
||||
|
||||
int DiskAnnContext::update_context(ContextType type, const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &measure,
|
||||
const DiskAnnEntity::Pointer &entity,
|
||||
uint32_t magic_num) {
|
||||
if (ailego_unlikely(type != type_)) {
|
||||
LOG_ERROR(
|
||||
"DiskAnnContext does not support shared by different type, "
|
||||
"src=%u dst=%u",
|
||||
type_, type);
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
magic_ = kInvalidMgic;
|
||||
|
||||
switch (type) {
|
||||
case kBuilderContext:
|
||||
LOG_ERROR("BuildContext does not support update");
|
||||
return IndexError_NotImplemented;
|
||||
|
||||
case kSearcherContext:
|
||||
break;
|
||||
|
||||
case kReducerContext:
|
||||
break;
|
||||
|
||||
default:
|
||||
LOG_ERROR("update context failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
entity_ = entity;
|
||||
dc_.update(measure, meta.dimension());
|
||||
magic_ = magic_num;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,372 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_context.h>
|
||||
#include "utility/topk_result_builder.h"
|
||||
#include "diskann_dist_calculator.h"
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_file_reader.h"
|
||||
#include "diskann_visit_filter.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
struct SearchStats {
|
||||
public:
|
||||
float total_us = 0;
|
||||
float io_us = 0;
|
||||
float cpu_us = 0;
|
||||
uint64_t disk_page_reads = 0;
|
||||
uint64_t io_num = 0;
|
||||
uint64_t dist_num = 0;
|
||||
uint64_t cache_hits = 0;
|
||||
uint64_t hop_num = 0;
|
||||
};
|
||||
|
||||
class DiskAnnContext : public IndexContext,
|
||||
public TopkResultBuilder<DiskAnnContext, TopkHeap> {
|
||||
friend class TopkResultBuilder<DiskAnnContext, TopkHeap>;
|
||||
|
||||
public:
|
||||
//! Index Context Pointer
|
||||
using Pointer = std::unique_ptr<DiskAnnContext>;
|
||||
|
||||
enum ContextType {
|
||||
kUnknownContext = 0,
|
||||
kSearcherContext = 1,
|
||||
kBuilderContext = 2,
|
||||
kReducerContext = 3
|
||||
};
|
||||
|
||||
//! Construct
|
||||
DiskAnnContext(const IndexMeta &meta, const IndexMetric::Pointer &measure,
|
||||
const DiskAnnEntity::Pointer &entity);
|
||||
|
||||
//! Destructor
|
||||
virtual ~DiskAnnContext();
|
||||
|
||||
public:
|
||||
//! Init
|
||||
int init(ContextType type, uint32_t graph_degree, uint32_t pq_chunk_num,
|
||||
uint32_t element_size);
|
||||
|
||||
//! Update context, the context may be shared by different searcher/streamer
|
||||
int update_context(ContextType type, const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &measure,
|
||||
const DiskAnnEntity::Pointer &entity, uint32_t magic_num);
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve result object for output
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(void) const override {
|
||||
return group_results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(
|
||||
size_t idx) const override {
|
||||
return group_results_[idx];
|
||||
}
|
||||
|
||||
virtual uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
void set_magic(uint32_t magic) {
|
||||
magic_ = magic;
|
||||
}
|
||||
|
||||
//! Set mode of debug
|
||||
virtual void set_debug_mode(bool enable) override {
|
||||
debug_mode_ = enable;
|
||||
}
|
||||
|
||||
//! Retrieve mode of debug
|
||||
virtual bool debug_mode(void) const override {
|
||||
return debug_mode_;
|
||||
}
|
||||
|
||||
//! Retrieve string of debug
|
||||
virtual std::string debug_string(void) const override {
|
||||
return std::string("");
|
||||
}
|
||||
|
||||
//! Update the parameters of context
|
||||
virtual int update(const ailego::Params & /*params*/) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline DistCalculator &dist_calculator() {
|
||||
return dc_;
|
||||
}
|
||||
|
||||
public:
|
||||
//! Set topk of search result
|
||||
void set_topk(uint32_t val) override {
|
||||
topk_ = val;
|
||||
topk_heap_.limit(val);
|
||||
}
|
||||
|
||||
void set_list_size(uint32_t list_size) {
|
||||
list_size_ = list_size;
|
||||
}
|
||||
|
||||
void set_fetch_vector(bool v) override {
|
||||
fetch_vector_ = v;
|
||||
}
|
||||
|
||||
//! Get topk
|
||||
inline uint32_t topk() const override {
|
||||
return topk_;
|
||||
}
|
||||
|
||||
inline uint32_t list_size() const {
|
||||
return list_size_;
|
||||
}
|
||||
|
||||
inline void reset_query(const void *query) {
|
||||
memcpy(query_, query, element_size_);
|
||||
memcpy(query_rotated_, query, element_size_);
|
||||
|
||||
dc_.reset_query(query);
|
||||
}
|
||||
|
||||
inline TopkHeap &topk_heap() {
|
||||
return topk_heap_;
|
||||
}
|
||||
|
||||
inline void *query() {
|
||||
return query_;
|
||||
}
|
||||
|
||||
inline void *query_rotated() {
|
||||
return query_rotated_;
|
||||
}
|
||||
|
||||
inline float *pq_table_dist_buffer() {
|
||||
return pq_table_dist_buffer_;
|
||||
}
|
||||
|
||||
inline void *pq_coord_buffer() {
|
||||
return pq_coord_buffer_;
|
||||
}
|
||||
|
||||
inline void *coord_buffer() {
|
||||
return coord_buffer_;
|
||||
}
|
||||
|
||||
inline void *sector_buffer() {
|
||||
return sector_buffer_;
|
||||
}
|
||||
|
||||
inline IOContext &io_ctx() {
|
||||
return io_ctx_;
|
||||
}
|
||||
|
||||
inline void resize_results(size_t size) {
|
||||
if (group_by_search()) {
|
||||
group_results_.resize(size);
|
||||
} else {
|
||||
results_.resize(size);
|
||||
}
|
||||
}
|
||||
|
||||
inline bool error() const {
|
||||
return has_error_;
|
||||
}
|
||||
|
||||
inline void set_error(bool err) {
|
||||
has_error_ = err;
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
for (auto &it : results_) {
|
||||
it.clear();
|
||||
}
|
||||
|
||||
best_list_nodes_.clear();
|
||||
expanded_nodes_.clear();
|
||||
visit_filter_.clear();
|
||||
has_error_ = false;
|
||||
}
|
||||
|
||||
SearchStats &query_stats() {
|
||||
return query_stats_;
|
||||
}
|
||||
|
||||
const DiskAnnEntity &get_entity() const {
|
||||
return *entity_;
|
||||
}
|
||||
|
||||
NeighborPriorityQueue &best_list_nodes() {
|
||||
return best_list_nodes_;
|
||||
}
|
||||
|
||||
std::vector<Neighbor> &expanded_nodes() {
|
||||
return expanded_nodes_;
|
||||
}
|
||||
|
||||
std::vector<float> &occlude_factor() {
|
||||
return occlude_factor_;
|
||||
}
|
||||
|
||||
VisitFilter &visit_filter() {
|
||||
return visit_filter_;
|
||||
}
|
||||
|
||||
//! Reset context
|
||||
void reset(void) override {
|
||||
set_filter(nullptr);
|
||||
reset_threshold();
|
||||
set_fetch_vector(false);
|
||||
set_group_params(0, 0);
|
||||
reset_group_by();
|
||||
}
|
||||
|
||||
inline std::map<std::string, TopkHeap> &group_topk_heaps() {
|
||||
return group_topk_heaps_;
|
||||
}
|
||||
|
||||
//! Get group topk
|
||||
inline uint32_t group_topk() const {
|
||||
return group_topk_;
|
||||
}
|
||||
|
||||
//! Get group num
|
||||
inline uint32_t group_num() const {
|
||||
return group_num_;
|
||||
}
|
||||
|
||||
//! Get if group by search
|
||||
inline bool group_by_search() {
|
||||
return group_num_ > 0;
|
||||
}
|
||||
|
||||
//! Set group params
|
||||
void set_group_params(uint32_t group_num, uint32_t group_topk) override {
|
||||
group_num_ = group_num;
|
||||
group_topk_ = group_topk;
|
||||
|
||||
topk_ = group_topk_ * group_num_;
|
||||
|
||||
topk_heap_.limit(topk_);
|
||||
|
||||
group_topk_heaps_.clear();
|
||||
}
|
||||
|
||||
// topk_to_result / topk_to_single_result / topk_to_group_result are provided
|
||||
// by TopkResultBuilder via the hooks below (result_score,
|
||||
// emplace_result_doc).
|
||||
using TopkResultBuilder<DiskAnnContext, TopkHeap>::topk_to_result;
|
||||
|
||||
void set_to_result(uint32_t idx, const std::vector<diskann_id_t> &result_ids,
|
||||
const std::vector<float> &result_dists) {
|
||||
if (result_ids.size() != result_dists.size()) {
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t size = result_ids.size();
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
results_[idx].emplace_back(result_ids[i], result_dists[i], 0);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
//! Hooks required by TopkResultBuilder.
|
||||
std::vector<IndexDocumentList> &mutable_results() {
|
||||
return results_;
|
||||
}
|
||||
|
||||
std::vector<IndexGroupDocumentList> &mutable_group_results() {
|
||||
return group_results_;
|
||||
}
|
||||
|
||||
float result_score(const VectorInfo &info) const {
|
||||
return info.dist_;
|
||||
}
|
||||
|
||||
void emplace_result_doc(IndexDocumentList &docs, diskann_id_t id, float score,
|
||||
const VectorInfo &info) {
|
||||
if (fetch_vector_) {
|
||||
docs.emplace_back(entity_->get_key(id), score, id, info.vec_);
|
||||
} else {
|
||||
docs.emplace_back(entity_->get_key(id), score, id);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
constexpr static uint32_t kInvalidMgic = -1U;
|
||||
|
||||
uint32_t type_{kUnknownContext};
|
||||
|
||||
DistCalculator dc_;
|
||||
DiskAnnEntity::Pointer entity_;
|
||||
|
||||
uint32_t topk_{0};
|
||||
uint32_t magic_{0U};
|
||||
bool debug_mode_{false};
|
||||
uint32_t pq_chunk_num_{0};
|
||||
uint32_t element_size_{0};
|
||||
uint32_t element_rotated_size_{0};
|
||||
uint32_t list_size_{0};
|
||||
|
||||
TopkHeap topk_heap_{};
|
||||
|
||||
uint32_t group_topk_{0};
|
||||
uint32_t group_num_{0};
|
||||
std::map<std::string, TopkHeap> group_topk_heaps_{};
|
||||
|
||||
IOContext io_ctx_{0};
|
||||
SearchStats query_stats_;
|
||||
|
||||
float *pq_table_dist_buffer_{nullptr};
|
||||
void *pq_coord_buffer_{nullptr};
|
||||
void *query_{nullptr};
|
||||
void *query_rotated_{nullptr};
|
||||
void *coord_buffer_{nullptr};
|
||||
void *sector_buffer_{nullptr};
|
||||
|
||||
std::vector<IndexDocumentList> results_{};
|
||||
std::vector<IndexGroupDocumentList> group_results_{};
|
||||
|
||||
bool fetch_vector_{false};
|
||||
bool has_error_{false};
|
||||
|
||||
NeighborPriorityQueue best_list_nodes_;
|
||||
std::vector<Neighbor> expanded_nodes_;
|
||||
std::vector<float> occlude_factor_;
|
||||
|
||||
VisitFilter visit_filter_{};
|
||||
uint32_t filter_mode_{VisitFilter::ByteMap};
|
||||
float negative_probility_{DiskAnnEntity::kDefaultBFNegativeProbility};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <zvec/core/framework/index_context.h>
|
||||
#include <zvec/core/framework/index_meta.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DistCalculator {
|
||||
public:
|
||||
typedef std::shared_ptr<DistCalculator> Pointer;
|
||||
|
||||
public:
|
||||
//! Constructor
|
||||
DistCalculator(const DiskAnnEntity *entity,
|
||||
const IndexMetric::Pointer &measure, uint32_t dim)
|
||||
: entity_(entity),
|
||||
distance_(measure->distance()),
|
||||
query_(nullptr),
|
||||
dim_(dim),
|
||||
compare_cnt_(0) {}
|
||||
|
||||
void update(const IndexMetric::Pointer &measure, uint32_t dim) {
|
||||
distance_ = measure->distance();
|
||||
dim_ = dim;
|
||||
}
|
||||
|
||||
inline void update_distance(const IndexMetric::MatrixDistance &distance) {
|
||||
distance_ = distance;
|
||||
}
|
||||
|
||||
//! Reset query vector data
|
||||
inline void reset_query(const void *query) {
|
||||
error_ = false;
|
||||
query_ = query;
|
||||
}
|
||||
|
||||
//! Returns distance
|
||||
inline dist_t dist(const void *vec_lhs, const void *vec_rhs) {
|
||||
if (ailego_unlikely(vec_lhs == nullptr || vec_rhs == nullptr)) {
|
||||
LOG_ERROR("Nullptr of dense vector");
|
||||
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
float score{0.0f};
|
||||
distance_(vec_lhs, vec_rhs, dim_, &score);
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
//! Returns distance between query and vec.
|
||||
inline dist_t dist(const void *vec) {
|
||||
compare_cnt_++;
|
||||
|
||||
return dist(vec, query_);
|
||||
}
|
||||
|
||||
inline dist_t dist(diskann_id_t id) {
|
||||
compare_cnt_++;
|
||||
|
||||
const void *vec = entity_->get_vector(id);
|
||||
if (ailego_unlikely(vec == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, id=%u", id);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
return dist(vec, query_);
|
||||
}
|
||||
|
||||
inline dist_t dist(diskann_id_t lhs, diskann_id_t rhs) {
|
||||
compare_cnt_++;
|
||||
|
||||
const void *vec_lhs = entity_->get_vector(lhs);
|
||||
if (ailego_unlikely(vec_lhs == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, lhs id=%u", lhs);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
const void *vec_rhs = entity_->get_vector(rhs);
|
||||
if (ailego_unlikely(vec_rhs == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, rhs id=%u", rhs);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
return dist(vec_lhs, vec_rhs);
|
||||
}
|
||||
|
||||
dist_t operator()(const void *vec) {
|
||||
return dist(vec);
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
compare_cnt_ = 0;
|
||||
error_ = false;
|
||||
}
|
||||
|
||||
inline void clear_compare_cnt() {
|
||||
compare_cnt_ = 0;
|
||||
}
|
||||
|
||||
inline bool error() const {
|
||||
return error_;
|
||||
}
|
||||
|
||||
//! Get distances compute times
|
||||
inline uint32_t compare_cnt() const {
|
||||
return compare_cnt_;
|
||||
}
|
||||
|
||||
inline uint32_t dimension() const {
|
||||
return dim_;
|
||||
}
|
||||
|
||||
private:
|
||||
DistCalculator(const DistCalculator &) = delete;
|
||||
DistCalculator &operator=(const DistCalculator &) = delete;
|
||||
|
||||
private:
|
||||
const DiskAnnEntity *entity_;
|
||||
|
||||
IndexMetric::MatrixDistance distance_;
|
||||
const void *query_;
|
||||
uint32_t dim_;
|
||||
|
||||
uint32_t compare_cnt_; // record distance compute times
|
||||
bool error_{false};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,32 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
const std::string DiskAnnEntity::kDiskAnnVectorSegmentId = "diskann.vector";
|
||||
const std::string DiskAnnEntity::kDiskAnnMetaSegmentId = "diskann.meta";
|
||||
const std::string DiskAnnEntity::kDiskAnnPqMetaSegmentId = "diskann.pq_meta";
|
||||
const std::string DiskAnnEntity::kDiskAnnPqDataSegmentId = "diskann.pq_data";
|
||||
const std::string DiskAnnEntity::kDiskAnnDummpySegmentId = "diskann.dummy";
|
||||
const std::string DiskAnnEntity::kDiskAnnKeyMappingSegmentId =
|
||||
"diskann.key_mapping";
|
||||
const std::string DiskAnnEntity::kDiskAnnEntryPointSegmentId =
|
||||
"diskann.entrypoint";
|
||||
const std::string DiskAnnEntity::kDiskAnnKeySegmentId = "diskann.key";
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,244 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <utility>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
using dist_t = float;
|
||||
using diskann_key_t = uint64_t;
|
||||
using diskann_id_t = uint32_t;
|
||||
|
||||
constexpr diskann_id_t kInvalidId = static_cast<diskann_id_t>(-1);
|
||||
constexpr diskann_key_t kInvalidKey = static_cast<key_t>(-1);
|
||||
|
||||
struct VectorInfo {
|
||||
float dist_;
|
||||
std::string vec_;
|
||||
|
||||
VectorInfo() = default;
|
||||
VectorInfo(float dist, const std::string &vec) : dist_{dist}, vec_{vec} {}
|
||||
VectorInfo(float dist, std::string &&vec)
|
||||
: dist_{dist}, vec_{std::move(vec)} {}
|
||||
};
|
||||
|
||||
/*! Key Value Vecotr Heap Comparer
|
||||
*/
|
||||
struct KeyValueVectorHeapComparer {
|
||||
//! Function call
|
||||
bool operator()(const std::pair<diskann_id_t, VectorInfo> &lhs,
|
||||
const std::pair<diskann_id_t, VectorInfo> &rhs) const {
|
||||
return compare_(lhs.second.dist_, rhs.second.dist_);
|
||||
}
|
||||
|
||||
private:
|
||||
std::less<float> compare_;
|
||||
};
|
||||
|
||||
/*! Key Value Vector Heap
|
||||
*/
|
||||
using TopkHeap = ailego::Heap<std::pair<diskann_id_t, VectorInfo>,
|
||||
KeyValueVectorHeapComparer>;
|
||||
|
||||
struct DiskAnnMetaHeader {
|
||||
public:
|
||||
uint64_t doc_cnt;
|
||||
uint64_t ndims;
|
||||
uint64_t medoid;
|
||||
uint64_t max_node_size;
|
||||
uint64_t max_degree;
|
||||
uint64_t node_per_sector;
|
||||
uint64_t vamana_frozen_num;
|
||||
uint64_t vamana_frozen_loc;
|
||||
uint64_t append_reorder_data;
|
||||
uint64_t index_size;
|
||||
uint8_t reserved[4016]; /// pad DiskAnnMetaHeader to 4096 bytes
|
||||
|
||||
DiskAnnMetaHeader() {
|
||||
clear();
|
||||
}
|
||||
|
||||
DiskAnnMetaHeader(const DiskAnnMetaHeader &header) {
|
||||
memcpy(this, &header, sizeof(header));
|
||||
}
|
||||
|
||||
DiskAnnMetaHeader &operator=(const DiskAnnMetaHeader &header) {
|
||||
if (this != &header) {
|
||||
memcpy(this, &header, sizeof(header));
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
void reset() {
|
||||
doc_cnt = 0U;
|
||||
}
|
||||
|
||||
void clear() {
|
||||
memset(this, 0, sizeof(DiskAnnMetaHeader));
|
||||
}
|
||||
};
|
||||
|
||||
struct DiskAnnPqMeta {
|
||||
public:
|
||||
uint64_t full_pivot_data_size{0};
|
||||
uint64_t centroid_data_size{0};
|
||||
uint64_t chunk_offsets_size{0};
|
||||
uint64_t chunk_num{0};
|
||||
uint8_t reserved[128];
|
||||
|
||||
DiskAnnPqMeta() {
|
||||
clear();
|
||||
}
|
||||
|
||||
DiskAnnPqMeta(const DiskAnnPqMeta &meta) {
|
||||
memcpy(this, &meta, sizeof(meta));
|
||||
}
|
||||
|
||||
DiskAnnPqMeta &operator=(const DiskAnnPqMeta &meta) {
|
||||
memcpy(this, &meta, sizeof(meta));
|
||||
return *this;
|
||||
}
|
||||
|
||||
void clear() {
|
||||
memset(this, 0, sizeof(DiskAnnPqMeta));
|
||||
}
|
||||
};
|
||||
|
||||
static_assert(sizeof(DiskAnnMetaHeader) == 4096,
|
||||
"DiskAnnMetaHeader size must stay 4096 bytes (on-disk format)");
|
||||
|
||||
static_assert(sizeof(DiskAnnPqMeta) % 32 == 0,
|
||||
"DiskAnnPqMeta size must be a multiple of 32 bytes");
|
||||
|
||||
class DiskAnnEntity {
|
||||
public:
|
||||
DiskAnnEntity() = default;
|
||||
virtual ~DiskAnnEntity() = default;
|
||||
|
||||
//! Constructor
|
||||
DiskAnnEntity(const DiskAnnMetaHeader &meta_header,
|
||||
const DiskAnnPqMeta &pq_meta) {
|
||||
meta_header_ = meta_header;
|
||||
pq_meta_ = pq_meta;
|
||||
}
|
||||
|
||||
//! DiskAnnEntity Pointerd;
|
||||
typedef std::shared_ptr<DiskAnnEntity> Pointer;
|
||||
|
||||
public:
|
||||
static size_t AlignSize(size_t size) {
|
||||
return (size + 0xFFF) & (~0xFFF);
|
||||
}
|
||||
|
||||
public:
|
||||
virtual int add_vector(diskann_key_t /*key*/, const void * /*vec*/) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
virtual const void *get_vector(diskann_id_t /*id*/) const {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
virtual std::pair<uint32_t, const diskann_id_t *> get_neighbors(
|
||||
diskann_id_t /*id*/) const {
|
||||
return std::make_pair(0, nullptr);
|
||||
}
|
||||
|
||||
virtual int set_neighbors(
|
||||
diskann_id_t /*id*/, const std::vector<diskann_id_t> & /*neighbor_ids*/) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
virtual int add_neighbor(diskann_id_t /*id*/, diskann_id_t /*neighbor_id*/) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Get node id of primary key
|
||||
virtual diskann_id_t get_id(diskann_key_t key) const = 0;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual diskann_key_t get_key(diskann_id_t id) const = 0;
|
||||
|
||||
public:
|
||||
uint64_t max_node_size() const {
|
||||
return meta_header_.max_node_size;
|
||||
}
|
||||
|
||||
uint64_t medoid() const {
|
||||
return meta_header_.medoid;
|
||||
}
|
||||
|
||||
uint64_t *mutable_medoid() {
|
||||
return &meta_header_.medoid;
|
||||
}
|
||||
|
||||
uint64_t node_per_sector() const {
|
||||
return meta_header_.node_per_sector;
|
||||
}
|
||||
|
||||
uint64_t pq_chunk_num() {
|
||||
return pq_meta_.chunk_num;
|
||||
}
|
||||
|
||||
uint64_t doc_cnt() const {
|
||||
return meta_header_.doc_cnt;
|
||||
}
|
||||
|
||||
uint64_t *mutable_doc_cnt() {
|
||||
return &meta_header_.doc_cnt;
|
||||
}
|
||||
|
||||
uint64_t max_degree() {
|
||||
return meta_header_.max_degree;
|
||||
}
|
||||
|
||||
DiskAnnPqMeta *mutable_pq_meta() {
|
||||
return &pq_meta_;
|
||||
}
|
||||
|
||||
public:
|
||||
virtual const DiskAnnEntity::Pointer clone() const {
|
||||
LOG_ERROR("Update neighbors not implemented");
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
public:
|
||||
const static std::string kDiskAnnVectorSegmentId;
|
||||
const static std::string kDiskAnnMetaSegmentId;
|
||||
const static std::string kDiskAnnPqMetaSegmentId;
|
||||
const static std::string kDiskAnnPqDataSegmentId;
|
||||
const static std::string kDiskAnnDummpySegmentId;
|
||||
const static std::string kDiskAnnMappingSegmentId;
|
||||
const static std::string kDiskAnnKeyMappingSegmentId;
|
||||
const static std::string kDiskAnnEntryPointSegmentId;
|
||||
const static std::string kDiskAnnKeySegmentId;
|
||||
|
||||
constexpr static float kDefaultBFNegativeProbility = 0.001f;
|
||||
constexpr static float kDefaultGraphSlackFactor = 1.3f;
|
||||
constexpr static float kDefaultAlpha = 1.2f;
|
||||
constexpr static uint32_t kDefaultMaxOcclusionSize = 750;
|
||||
constexpr static uint32_t kDefaultMaxDegree = 100;
|
||||
constexpr static uint32_t kDefaultCompressBatchSize = 5000000;
|
||||
constexpr static uint32_t kRevision = 0U;
|
||||
|
||||
protected:
|
||||
DiskAnnMetaHeader meta_header_;
|
||||
DiskAnnPqMeta pq_meta_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,299 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_file_reader.h"
|
||||
#include <cassert>
|
||||
#include <cerrno>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <zvec/core/framework/index_logger.h>
|
||||
|
||||
#define MAX_EVENTS 1024
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
typedef struct io_event io_event_t;
|
||||
typedef struct iocb iocb_t;
|
||||
#endif
|
||||
|
||||
int setup_io_ctx(IOContext &ctx) {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
int ret = io_setup(MAX_EVENTS, &ctx);
|
||||
|
||||
return ret;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
int destroy_io_ctx(IOContext &ctx) {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
int ret = io_destroy(ctx);
|
||||
|
||||
return ret;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int execute_io_pread(int fd, std::vector<AlignedRead> &read_reqs) {
|
||||
for (auto &req : read_reqs) {
|
||||
ssize_t bytes_read = ::pread(fd, req.buf, req.len, req.offset);
|
||||
if (bytes_read < 0) {
|
||||
LOG_ERROR("pread failed; errno=%d, %s, offset=%lu, len=%lu", errno,
|
||||
::strerror(errno), (unsigned long)req.offset,
|
||||
(unsigned long)req.len);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
if ((size_t)bytes_read != req.len) {
|
||||
LOG_ERROR("pread short read; got=%zd, expected=%lu", bytes_read,
|
||||
(unsigned long)req.len);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int execute_io(IOContext ctx, int fd, std::vector<AlignedRead> &read_reqs,
|
||||
uint64_t n_retries = 0) {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
uint64_t iters = DiskAnnUtil::div_round_up(read_reqs.size(), MAX_EVENTS);
|
||||
|
||||
for (uint64_t iter = 0; iter < iters; iter++) {
|
||||
uint64_t n_ops = std::min((uint64_t)read_reqs.size() - (iter * MAX_EVENTS),
|
||||
(uint64_t)MAX_EVENTS);
|
||||
|
||||
std::vector<iocb_t *> cbs(n_ops, nullptr);
|
||||
std::vector<io_event_t> evts(n_ops);
|
||||
std::vector<struct iocb> cb(n_ops);
|
||||
for (uint64_t j = 0; j < n_ops; j++) {
|
||||
io_prep_pread(cb.data() + j, fd, read_reqs[j + iter * MAX_EVENTS].buf,
|
||||
read_reqs[j + iter * MAX_EVENTS].len,
|
||||
read_reqs[j + iter * MAX_EVENTS].offset);
|
||||
}
|
||||
|
||||
for (uint64_t i = 0; i < n_ops; i++) {
|
||||
cbs[i] = cb.data() + i;
|
||||
}
|
||||
|
||||
size_t n_tries = 0;
|
||||
// Phase 1: io_submit with retry.
|
||||
while (true) {
|
||||
int ret = io_submit(ctx, (int64_t)n_ops, cbs.data());
|
||||
if (ret == (int)n_ops) {
|
||||
break;
|
||||
}
|
||||
if ((ret == -EAGAIN || ret == -EINTR) && n_tries < n_retries) {
|
||||
n_tries++;
|
||||
continue;
|
||||
}
|
||||
LOG_WARN(
|
||||
"io_submit failed; returned: %d, expected=%lu. falling back to "
|
||||
"pread",
|
||||
ret, n_ops);
|
||||
return execute_io_pread(fd, read_reqs);
|
||||
}
|
||||
|
||||
// Phase 2: io_getevents with retry (never re-submits).
|
||||
n_tries = 0;
|
||||
while (true) {
|
||||
int ret = io_getevents(ctx, (int64_t)n_ops, (int64_t)n_ops, evts.data(),
|
||||
nullptr);
|
||||
if (ret == (int)n_ops) {
|
||||
break;
|
||||
}
|
||||
if (ret == -EINTR && n_tries < n_retries) {
|
||||
n_tries++;
|
||||
continue;
|
||||
}
|
||||
LOG_WARN(
|
||||
"io_getevents failed; returned: %d, expected=%lu, errno=%d, %s, "
|
||||
"falling back to pread",
|
||||
ret, n_ops, errno, ::strerror(-ret));
|
||||
return execute_io_pread(fd, read_reqs);
|
||||
}
|
||||
|
||||
// Phase 3: verify each completed read (res must equal requested length).
|
||||
bool all_ok = true;
|
||||
for (uint64_t i = 0; i < n_ops; i++) {
|
||||
int64_t expected_len = read_reqs[i + iter * MAX_EVENTS].len;
|
||||
if ((int64_t)evts[i].res != expected_len) {
|
||||
LOG_WARN("aio request %zu failed: res=%ld, expected=%ld, offset=%zu",
|
||||
(size_t)i, (long)evts[i].res, (long)expected_len,
|
||||
(size_t)read_reqs[i + iter * MAX_EVENTS].offset);
|
||||
all_ok = false;
|
||||
}
|
||||
}
|
||||
if (!all_ok) {
|
||||
return execute_io_pread(fd, read_reqs);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
#else
|
||||
return execute_io_pread(fd, read_reqs);
|
||||
#endif
|
||||
}
|
||||
|
||||
LinuxAlignedFileReader::LinuxAlignedFileReader(int file_desc) {
|
||||
this->file_desc = file_desc;
|
||||
}
|
||||
|
||||
LinuxAlignedFileReader::LinuxAlignedFileReader() {
|
||||
this->file_desc = -1;
|
||||
}
|
||||
|
||||
LinuxAlignedFileReader::~LinuxAlignedFileReader() {
|
||||
deregister_all_threads();
|
||||
if (file_desc >= 0) {
|
||||
::close(file_desc);
|
||||
file_desc = -1;
|
||||
}
|
||||
}
|
||||
|
||||
IOContext &LinuxAlignedFileReader::get_ctx() {
|
||||
std::unique_lock<std::mutex> lk(ctx_mut);
|
||||
auto it = ctx_map.find(std::this_thread::get_id());
|
||||
if (it == ctx_map.end()) {
|
||||
LOG_ERROR("bad thread access; returning -1 as io_context_t");
|
||||
return this->bad_ctx;
|
||||
} else {
|
||||
return it->second;
|
||||
}
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::register_thread() {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
auto thread_id = std::this_thread::get_id();
|
||||
std::unique_lock<std::mutex> lk(ctx_mut);
|
||||
if (ctx_map.find(thread_id) != ctx_map.end()) {
|
||||
LOG_ERROR("multiple calls to register_thread from the same thread");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
IOContext ctx = nullptr;
|
||||
|
||||
int ret = io_setup(MAX_EVENTS, &ctx);
|
||||
if (ret != 0) {
|
||||
lk.unlock();
|
||||
if (ret == -EAGAIN) {
|
||||
LOG_ERROR(
|
||||
"io_setup failed with EAGAIN: Consider increasing "
|
||||
"/proc/sys/fs/aio-max-nr");
|
||||
} else {
|
||||
LOG_ERROR("io_setup failed; returned: %d, %s", ret, ::strerror(-ret));
|
||||
;
|
||||
}
|
||||
} else {
|
||||
LOG_INFO("allocating ctx: %lu", (uint64_t)ctx);
|
||||
|
||||
ctx_map[thread_id] = ctx;
|
||||
}
|
||||
|
||||
lk.unlock();
|
||||
#endif
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::deregister_thread() {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
auto thread_id = std::this_thread::get_id();
|
||||
IOContext ctx;
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(ctx_mut);
|
||||
auto it = ctx_map.find(thread_id);
|
||||
if (it == ctx_map.end()) {
|
||||
LOG_ERROR("deregister_thread: thread not registered");
|
||||
return;
|
||||
}
|
||||
ctx = it->second;
|
||||
ctx_map.erase(it);
|
||||
}
|
||||
|
||||
// io_destroy is a syscall; keep it outside the lock to avoid blocking others
|
||||
io_destroy(ctx);
|
||||
LOG_INFO("returned ctx from thread");
|
||||
#endif
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::deregister_all_threads() {
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
std::unique_lock<std::mutex> lk(ctx_mut);
|
||||
for (auto x = ctx_map.begin(); x != ctx_map.end(); x++) {
|
||||
IOContext ctx = x->second;
|
||||
io_destroy(ctx);
|
||||
}
|
||||
ctx_map.clear();
|
||||
#endif
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::open(const std::string &fname) {
|
||||
int flags = O_RDONLY;
|
||||
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
flags |= O_DIRECT | O_LARGEFILE;
|
||||
#endif
|
||||
|
||||
this->file_desc = ::open(fname.c_str(), flags);
|
||||
|
||||
#if defined(__linux__) || defined(__linux)
|
||||
// O_DIRECT may not be supported on all filesystems (e.g. tmpfs, overlay).
|
||||
// Fall back to regular buffered I/O when it fails.
|
||||
if (this->file_desc == -1) {
|
||||
LOG_WARN(
|
||||
"open with O_DIRECT failed for %s (errno=%d: %s), "
|
||||
"falling back to buffered I/O",
|
||||
fname.c_str(), errno, ::strerror(errno));
|
||||
this->file_desc = ::open(fname.c_str(), O_RDONLY | O_LARGEFILE);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (this->file_desc == -1) {
|
||||
LOG_ERROR("Failed to open file: %s (errno=%d: %s)", fname.c_str(), errno,
|
||||
::strerror(errno));
|
||||
}
|
||||
|
||||
LOG_INFO("Opened file : %s", fname.c_str());
|
||||
}
|
||||
|
||||
void LinuxAlignedFileReader::close() {
|
||||
if (file_desc >= 0) {
|
||||
::close(file_desc);
|
||||
file_desc = -1;
|
||||
}
|
||||
}
|
||||
|
||||
int LinuxAlignedFileReader::read(std::vector<AlignedRead> &read_reqs,
|
||||
IOContext &ctx, bool async) {
|
||||
if (async == true) {
|
||||
LOG_WARN("Async currently not supported");
|
||||
}
|
||||
|
||||
if (this->file_desc == -1) {
|
||||
LOG_ERROR("Attempt to read from invalid file descriptor");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
int ret = execute_io(ctx, this->file_desc, read_reqs);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,104 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#define MAX_IO_DEPTH 128
|
||||
|
||||
#include <fcntl.h>
|
||||
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
#include <libaio.h>
|
||||
#endif
|
||||
|
||||
#include <unistd.h>
|
||||
#include <atomic>
|
||||
#include <vector>
|
||||
#include <zvec/core/framework/index_context.h>
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
#if (defined(__linux) || defined(__linux__))
|
||||
typedef io_context_t IOContext;
|
||||
#else
|
||||
typedef uint32_t IOContext;
|
||||
#endif
|
||||
|
||||
int setup_io_ctx(IOContext &ctx);
|
||||
int destroy_io_ctx(IOContext &ctx);
|
||||
|
||||
struct AlignedRead {
|
||||
uint64_t offset;
|
||||
uint64_t len;
|
||||
void *buf;
|
||||
|
||||
AlignedRead() : offset(0), len(0), buf(nullptr) {}
|
||||
|
||||
AlignedRead(uint64_t offset, uint64_t len, void *buf)
|
||||
: offset(offset), len(len), buf(buf) {
|
||||
ailego_assert(static_cast<size_t>(offset) % 512 == 0);
|
||||
ailego_assert(static_cast<size_t>(len) % 512 == 0);
|
||||
ailego_assert(reinterpret_cast<size_t>(buf) % 512 == 0);
|
||||
}
|
||||
};
|
||||
|
||||
class AlignedFileReader {
|
||||
protected:
|
||||
std::map<std::thread::id, IOContext> ctx_map;
|
||||
std::mutex ctx_mut;
|
||||
|
||||
public:
|
||||
virtual IOContext &get_ctx() = 0;
|
||||
|
||||
virtual ~AlignedFileReader() {};
|
||||
|
||||
virtual void register_thread() = 0;
|
||||
virtual void deregister_thread() = 0;
|
||||
virtual void deregister_all_threads() = 0;
|
||||
|
||||
virtual void open(const std::string &fname) = 0;
|
||||
virtual void close() = 0;
|
||||
|
||||
virtual int read(std::vector<AlignedRead> &read_reqs, IOContext &ctx,
|
||||
bool async = false) = 0;
|
||||
};
|
||||
|
||||
class LinuxAlignedFileReader : public AlignedFileReader {
|
||||
private:
|
||||
uint64_t file_sz;
|
||||
int file_desc;
|
||||
|
||||
IOContext bad_ctx = (IOContext)-1;
|
||||
|
||||
public:
|
||||
LinuxAlignedFileReader();
|
||||
LinuxAlignedFileReader(int file_desc);
|
||||
~LinuxAlignedFileReader();
|
||||
|
||||
public:
|
||||
IOContext &get_ctx();
|
||||
|
||||
void register_thread();
|
||||
void deregister_thread();
|
||||
void deregister_all_threads();
|
||||
void open(const std::string &fname);
|
||||
void close();
|
||||
|
||||
int read(std::vector<AlignedRead> &read_reqs, IOContext &ctx,
|
||||
bool async = false);
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,325 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <fstream>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
struct DiskAnnIndexHolderMeta {
|
||||
uint32_t element_size_;
|
||||
uint32_t key_size_;
|
||||
uint32_t sector_size_;
|
||||
uint32_t doc_cnt_;
|
||||
uint8_t reserve_[];
|
||||
};
|
||||
|
||||
class DiskAnnIndexHolder : public IndexHolder {
|
||||
public:
|
||||
typedef std::shared_ptr<DiskAnnIndexHolder> Pointer;
|
||||
|
||||
public:
|
||||
enum Status { STATUS_UNINITED = 0, STATUS_WRITE = 1, STATUS_READ = 2 };
|
||||
|
||||
public:
|
||||
static constexpr uint32_t kDataSectorSize = 128 * 1024;
|
||||
static constexpr uint32_t kMetaSectorSize = 4096;
|
||||
|
||||
public:
|
||||
inline static uint32_t get_sector_id(uint32_t id, uint32_t sector_vec_num) {
|
||||
return id / sector_vec_num;
|
||||
}
|
||||
|
||||
inline static uint32_t get_sector_offset(uint32_t id, uint32_t sector_vec_num,
|
||||
uint32_t data_size) {
|
||||
return (id % sector_vec_num) * data_size;
|
||||
}
|
||||
|
||||
public:
|
||||
/*! Random Access Index Holder Iterator
|
||||
*/
|
||||
class Iterator : public IndexHolder::Iterator {
|
||||
public:
|
||||
//! Index Holder Iterator Pointer
|
||||
typedef std::unique_ptr<Iterator> Pointer;
|
||||
|
||||
//! Constructor
|
||||
Iterator(DiskAnnIndexHolder *owner)
|
||||
: holder_(owner), sector_id_{0}, sector_offset_{0} {
|
||||
path_ = holder_->path();
|
||||
data_size_ = holder_->data_size();
|
||||
data_sector_size_ = holder_->data_sector_size();
|
||||
meta_sector_size_ = holder_->meta_sector_size();
|
||||
|
||||
sector_buffer_.resize(data_sector_size_);
|
||||
|
||||
sector_vec_num_ = data_sector_size_ / data_size_;
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
virtual ~Iterator(void) {
|
||||
if (file_.is_open()) {
|
||||
file_.close();
|
||||
}
|
||||
}
|
||||
|
||||
int init() {
|
||||
file_.open(path_, std::ios::in);
|
||||
if (!file_.is_open()) {
|
||||
LOG_ERROR("file can not create, %s", path_.c_str());
|
||||
return IndexError_OpenFile;
|
||||
}
|
||||
|
||||
file_.seekg(meta_sector_size_);
|
||||
|
||||
read_sector();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
const void *data(void) const override {
|
||||
const uint8_t *data_ptr =
|
||||
reinterpret_cast<const uint8_t *>(sector_buffer_.data());
|
||||
return data_ptr + sector_offset_ + sizeof(diskann_key_t);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
bool is_valid(void) const override {
|
||||
return id_ < holder_->count();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
uint64_t key(void) const override {
|
||||
const uint8_t *data_ptr =
|
||||
reinterpret_cast<const uint8_t *>(sector_buffer_.data());
|
||||
uint64_t key =
|
||||
*reinterpret_cast<const uint64_t *>(data_ptr + sector_offset_);
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
void next(void) override {
|
||||
++id_;
|
||||
|
||||
uint32_t sector_id = get_sector_id(id_, sector_vec_num_);
|
||||
if (sector_id > sector_id_) {
|
||||
file_.seekg(sector_id * data_sector_size_ + meta_sector_size_);
|
||||
read_sector();
|
||||
sector_id_ = sector_id;
|
||||
}
|
||||
|
||||
sector_offset_ = get_sector_offset(id_, sector_vec_num_, data_size_);
|
||||
}
|
||||
|
||||
int read_sector() {
|
||||
file_.read(&((sector_buffer_)[0]), data_sector_size_);
|
||||
if (!file_) {
|
||||
LOG_ERROR("Failed to read sector from file: %s", path_.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
DiskAnnIndexHolder *holder_{nullptr};
|
||||
std::string path_;
|
||||
std::ifstream file_;
|
||||
uint32_t sector_id_{0};
|
||||
std::string sector_buffer_;
|
||||
uint32_t id_{0};
|
||||
uint32_t data_size_{0};
|
||||
uint32_t sector_offset_{0};
|
||||
uint32_t data_sector_size_{0};
|
||||
uint32_t meta_sector_size_{0};
|
||||
uint32_t sector_vec_num_{0};
|
||||
};
|
||||
|
||||
public:
|
||||
DiskAnnIndexHolder(IndexMeta &meta, std::string &path) {
|
||||
path_ = path;
|
||||
|
||||
data_size_ = meta.element_size() + sizeof(diskann_key_t);
|
||||
dimension_ = meta.dimension();
|
||||
type_ = meta.data_type();
|
||||
|
||||
element_size_ = meta.element_size();
|
||||
sector_vec_num_ = data_sector_size_ / data_size_;
|
||||
padding_size_ = data_sector_size_ - sector_vec_num_ * data_size_;
|
||||
sector_buffer_.resize(data_sector_size_);
|
||||
sector_internal_id_ = 0;
|
||||
}
|
||||
|
||||
~DiskAnnIndexHolder() override {
|
||||
if (file_.is_open()) {
|
||||
file_.close();
|
||||
}
|
||||
}
|
||||
|
||||
//! Init
|
||||
int init() {
|
||||
file_.open(path_, std::ios::out | std::ios::trunc);
|
||||
|
||||
if (!file_.is_open()) {
|
||||
LOG_ERROR("file can not create, %s", path_.c_str());
|
||||
return IndexError_OpenFile;
|
||||
}
|
||||
|
||||
DiskAnnIndexHolderMeta holder_meta;
|
||||
holder_meta.element_size_ = element_size_;
|
||||
holder_meta.key_size_ = sizeof(diskann_key_t);
|
||||
holder_meta.sector_size_ = data_sector_size_;
|
||||
|
||||
std::vector<uint8_t> empty_sector;
|
||||
empty_sector.resize(meta_sector_size_);
|
||||
|
||||
std::memset(&(empty_sector[0]), 0, meta_sector_size_);
|
||||
std::memcpy(&(empty_sector[0]), &holder_meta,
|
||||
sizeof(DiskAnnIndexHolderMeta));
|
||||
|
||||
file_.write(reinterpret_cast<const char *>(&(empty_sector[0])),
|
||||
meta_sector_size_);
|
||||
if (!file_) {
|
||||
LOG_ERROR("Failed to write meta sector to file: %s", path_.c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
status_ = STATUS_WRITE;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int close() {
|
||||
if (sector_internal_id_ != 0) {
|
||||
file_.write(reinterpret_cast<const char *>(&(sector_buffer_[0])),
|
||||
data_sector_size_);
|
||||
}
|
||||
|
||||
file_.close();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve count of elements in holder (-1 indicates unknown)
|
||||
size_t count(void) const override {
|
||||
return count_;
|
||||
}
|
||||
|
||||
//! Retrieve dimension
|
||||
size_t dimension(void) const override {
|
||||
return dimension_;
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return type_;
|
||||
}
|
||||
|
||||
//! Retrieve element size in bytes
|
||||
size_t element_size(void) const override {
|
||||
return element_size_;
|
||||
}
|
||||
|
||||
//! Retrieve if it can multi-pass
|
||||
bool multipass(void) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Create a new iterator
|
||||
IndexHolder::Iterator::Pointer create_iterator(void) override {
|
||||
auto pointer = std::make_unique<DiskAnnIndexHolder::Iterator>(this);
|
||||
|
||||
if (pointer->init() != 0) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return pointer;
|
||||
}
|
||||
|
||||
int emplace(uint64_t pkey, const void *vec) {
|
||||
if (status_ != STATUS_WRITE) {
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
uint8_t *data_ptr = reinterpret_cast<uint8_t *>(&(sector_buffer_[0])) +
|
||||
sector_internal_id_ * data_size_;
|
||||
std::memcpy(data_ptr, &pkey, sizeof(diskann_key_t));
|
||||
std::memcpy(data_ptr + sizeof(diskann_key_t), vec, element_size_);
|
||||
|
||||
sector_internal_id_++;
|
||||
if (sector_internal_id_ >= sector_vec_num_) {
|
||||
std::memset(data_ptr + data_size_, 0, padding_size_);
|
||||
|
||||
file_.write(reinterpret_cast<const char *>(&(sector_buffer_[0])),
|
||||
data_sector_size_);
|
||||
|
||||
sector_internal_id_ = 0;
|
||||
sector_id_++;
|
||||
}
|
||||
|
||||
count_++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t data_sector_size() {
|
||||
return data_sector_size_;
|
||||
}
|
||||
|
||||
uint32_t meta_sector_size() {
|
||||
return meta_sector_size_;
|
||||
}
|
||||
|
||||
uint32_t data_size() {
|
||||
return data_size_;
|
||||
}
|
||||
|
||||
uint32_t *mutable_sector_id() {
|
||||
return §or_id_;
|
||||
}
|
||||
|
||||
uint32_t sector_id() {
|
||||
return sector_id_;
|
||||
}
|
||||
|
||||
std::string &path() {
|
||||
return path_;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string path_;
|
||||
std::ofstream file_;
|
||||
uint32_t element_size_{0};
|
||||
uint32_t dimension_{0};
|
||||
IndexMeta::DataType type_{IndexMeta::DataType::DT_UNDEFINED};
|
||||
uint32_t sector_vec_num_{0};
|
||||
uint32_t data_size_{0};
|
||||
uint32_t padding_size_{0};
|
||||
uint32_t meta_sector_size_{DiskAnnIndexHolder::kMetaSectorSize};
|
||||
uint32_t data_sector_size_{DiskAnnIndexHolder::kDataSectorSize};
|
||||
std::string sector_buffer_;
|
||||
uint32_t sector_internal_id_{0};
|
||||
uint32_t sector_id_{0};
|
||||
uint32_t count_{0};
|
||||
uint32_t status_{STATUS_UNINITED};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,120 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_provider.h>
|
||||
#include <zvec/core/framework/index_searcher.h>
|
||||
#include <zvec/core/framework/index_streamer.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! IndexProvider implementation backed by a DiskAnn entity.
|
||||
//!
|
||||
//! Used by ``MixedStreamerReducer`` during segment merge: the reducer needs
|
||||
//! to walk every vector held by a source DiskAnn streamer and feed it into
|
||||
//! the merge target. Vectors are read on demand from the entity's on-disk
|
||||
//! vector segment via ``DiskAnnEntity::get_vector(id)``.
|
||||
class DiskAnnIndexProvider : public IndexProvider {
|
||||
public:
|
||||
DiskAnnIndexProvider(const IndexMeta &meta,
|
||||
const DiskAnnEntity::Pointer &entity,
|
||||
const std::string &owner)
|
||||
: meta_(meta), entity_(entity), owner_class_(owner) {}
|
||||
|
||||
DiskAnnIndexProvider(const DiskAnnIndexProvider &) = delete;
|
||||
DiskAnnIndexProvider &operator=(const DiskAnnIndexProvider &) = delete;
|
||||
|
||||
public:
|
||||
IndexProvider::Iterator::Pointer create_iterator() override {
|
||||
return IndexProvider::Iterator::Pointer(new (std::nothrow)
|
||||
Iterator(entity_));
|
||||
}
|
||||
|
||||
size_t count(void) const override {
|
||||
return entity_->doc_cnt();
|
||||
}
|
||||
|
||||
size_t dimension(void) const override {
|
||||
return meta_.dimension();
|
||||
}
|
||||
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return meta_.data_type();
|
||||
}
|
||||
|
||||
size_t element_size(void) const override {
|
||||
return meta_.element_size();
|
||||
}
|
||||
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
diskann_id_t id = entity_->get_id(static_cast<diskann_key_t>(key));
|
||||
if (id == kInvalidId) {
|
||||
return nullptr;
|
||||
}
|
||||
return entity_->get_vector(id);
|
||||
}
|
||||
|
||||
const std::string &owner_class(void) const override {
|
||||
return owner_class_;
|
||||
}
|
||||
|
||||
private:
|
||||
class Iterator : public IndexProvider::Iterator {
|
||||
public:
|
||||
explicit Iterator(const DiskAnnEntity::Pointer &entity)
|
||||
: entity_(entity), cur_id_(0U) {
|
||||
cur_id_ = next_valid_id(0U);
|
||||
}
|
||||
|
||||
const void *data(void) const override {
|
||||
return entity_->get_vector(cur_id_);
|
||||
}
|
||||
|
||||
bool is_valid(void) const override {
|
||||
return cur_id_ < static_cast<diskann_id_t>(entity_->doc_cnt());
|
||||
}
|
||||
|
||||
uint64_t key(void) const override {
|
||||
return static_cast<uint64_t>(entity_->get_key(cur_id_));
|
||||
}
|
||||
|
||||
void next(void) override {
|
||||
cur_id_ = next_valid_id(cur_id_ + 1);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Skip ids that map to ``kInvalidKey`` (deleted / never populated slots).
|
||||
diskann_id_t next_valid_id(diskann_id_t start_id) const {
|
||||
const auto total = static_cast<diskann_id_t>(entity_->doc_cnt());
|
||||
for (diskann_id_t i = start_id; i < total; ++i) {
|
||||
if (entity_->get_key(i) != kInvalidKey) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
DiskAnnEntity::Pointer entity_;
|
||||
diskann_id_t cur_id_;
|
||||
};
|
||||
|
||||
IndexMeta meta_;
|
||||
DiskAnnEntity::Pointer entity_;
|
||||
std::string owner_class_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,109 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_file_reader.h"
|
||||
#include "diskann_pq_table.h"
|
||||
#include "diskann_searcher_entity.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnIndexer {
|
||||
public:
|
||||
typedef std::shared_ptr<DiskAnnIndexer> Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnIndexer(const IndexMeta &meta);
|
||||
~DiskAnnIndexer();
|
||||
|
||||
public:
|
||||
int init(DiskAnnSearcherEntity &entity);
|
||||
int load_cache_list(const std::vector<diskann_id_t> &node_list);
|
||||
|
||||
void cache_bfs_levels(uint64_t num_nodes_to_cache,
|
||||
std::vector<diskann_id_t> &node_list);
|
||||
|
||||
int cached_beam_search(DiskAnnContext *ctx);
|
||||
int cached_beam_search_by_group(DiskAnnContext *ctx);
|
||||
|
||||
int cached_beam_search_in_mem(DiskAnnContext *ctx);
|
||||
|
||||
int knn_search(DiskAnnContext *ctx);
|
||||
int linear_search(DiskAnnContext *ctx);
|
||||
int keys_search(const std::vector<diskann_key_t> &keys, DiskAnnContext *ctx);
|
||||
|
||||
int get_vector(diskann_id_t id, IndexContext::Pointer &context,
|
||||
std::string &vector);
|
||||
|
||||
diskann_key_t get_key(diskann_id_t id) const;
|
||||
diskann_id_t get_id(diskann_key_t key) const;
|
||||
|
||||
//! Copy element_size() bytes from src into a new vector value string
|
||||
std::string make_vector_copy(const void *src) const {
|
||||
return std::string(static_cast<const char *>(src), meta_.element_size());
|
||||
}
|
||||
|
||||
std::vector<bool> read_nodes(
|
||||
const std::vector<diskann_id_t> &node_ids,
|
||||
std::vector<void *> &coord_buffers,
|
||||
std::vector<std::pair<uint32_t, diskann_id_t *>> &nbr_buffers);
|
||||
|
||||
protected:
|
||||
int use_medroids_data_as_centroids();
|
||||
|
||||
private:
|
||||
DiskAnnSearcherEntity *entity_;
|
||||
|
||||
IndexStorage::Pointer storage_{};
|
||||
IndexMeta meta_;
|
||||
|
||||
uint32_t max_degree_{0};
|
||||
uint32_t node_per_sector_{0};
|
||||
uint32_t max_node_size_{0};
|
||||
uint64_t pq_chunk_num_{0};
|
||||
uint64_t disk_bytes_per_point_{0};
|
||||
uint64_t aligned_dim_{0};
|
||||
uint64_t index_segment_offset_{0};
|
||||
uint64_t sector_num_per_node_{0};
|
||||
|
||||
float *centroid_data_{nullptr};
|
||||
|
||||
diskann_id_t medoid_;
|
||||
std::vector<diskann_id_t> entrypoints_;
|
||||
|
||||
std::shared_ptr<LinuxAlignedFileReader> reader_{nullptr};
|
||||
|
||||
PQTable::Pointer pq_table_;
|
||||
|
||||
IOContext init_ctx_{0};
|
||||
|
||||
std::vector<diskann_id_t> neighbor_cache_buffer_;
|
||||
void *coord_cache_buf_{nullptr};
|
||||
|
||||
std::map<diskann_id_t, void *> coord_cache_;
|
||||
std::map<diskann_id_t, std::pair<uint32_t, diskann_id_t *>> neighbor_cache_;
|
||||
|
||||
uint32_t beam_width_{2};
|
||||
uint32_t io_limit_{std::numeric_limits<uint32_t>::max()};
|
||||
|
||||
uint64_t doc_cnt_{0};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,53 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
static const std::string PARAM_DISKANN_BUILDER_MAX_DEGREE(
|
||||
"zvec.diskann.builder.max_degree");
|
||||
static const std::string PARAM_DISKANN_BUILDER_LIST_SIZE(
|
||||
"zvec.diskann.builder.list_size");
|
||||
static const std::string PARAM_DISKANN_BUILDER_MEMORY_LIMIT(
|
||||
"zvec.diskann.builder.memory_limit");
|
||||
static const std::string PARAM_DISKANN_BUILDER_MEMORY_BUDGET(
|
||||
"zvec.diskann.builder.memory_budget");
|
||||
static const std::string PARAM_DISKANN_BUILDER_DISK_PQ_DIM(
|
||||
"zvec.diskann.builder.disk_pq_dim");
|
||||
static const std::string PARAM_DISKANN_BUILDER_THREAD_COUNT(
|
||||
"zvec.diskann.builder.thread_count");
|
||||
static const std::string PARAM_DISKANN_BUILDER_MAX_TRAIN_SAMPLE_COUNT(
|
||||
"zvec.diskann.builder.max_train_sample_count");
|
||||
static const std::string PARAM_DISKANN_BUILDER_TRAIN_SAMPLE_RATIO(
|
||||
"zvec.diskann.builder.train_sample_ratio");
|
||||
static const std::string PARAM_DISKANN_BUILDER_MAX_PQ_CHUNK_NUM(
|
||||
"zvec.diskann.builder.max_pq_chunk_num");
|
||||
|
||||
static const std::string PARAM_DISKANN_SEARCHER_LIST_SIZE(
|
||||
"zvec.diskann.searcher.list_size");
|
||||
static const std::string PARAM_DISKANN_SEARCHER_CACHE_NODE_NUM(
|
||||
"zvec.diskann.searcher.cache_node_num");
|
||||
|
||||
static const std::string PARAM_DISKANN_REDUCER_INDEX_NAME(
|
||||
"zvec.diskann.reducer.index_name");
|
||||
static const std::string PARAM_DISKANN_REDUCER_WORKING_PATH(
|
||||
"zvec.diskann.reducer.working_path");
|
||||
static const std::string PARAM_DISKANN_REDUCER_NUM_OF_ADD_THREADS(
|
||||
"zvec.diskann.reducer.num_of_add_threads");
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,117 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_pq_table.h"
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
PQTable::PQTable(const IndexMeta &meta, uint32_t chunk_num)
|
||||
: chunk_num_(chunk_num) {
|
||||
meta_ = meta;
|
||||
|
||||
if (meta.metric_name() == "Cosine") {
|
||||
if (meta.data_type() == IndexMeta::DataType::DT_FP32) {
|
||||
meta_.set_dimension(meta.dimension() - 1);
|
||||
} else {
|
||||
meta_.set_dimension(meta.dimension() - 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PQTable::~PQTable() {}
|
||||
|
||||
int PQTable::init(std::vector<uint8_t> &full_pivot_data,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint32_t> &chunk_offsets,
|
||||
std::vector<uint8_t> &pq_data) {
|
||||
full_pivot_data_ = std::move(full_pivot_data);
|
||||
centroid_ = std::move(centroid);
|
||||
chunk_offsets_ = std::move(chunk_offsets);
|
||||
pq_data_ = std::move(pq_data);
|
||||
|
||||
// alloc and compute transpose
|
||||
transposed_tables_.resize(kPQCentroidNum * meta_.element_size());
|
||||
|
||||
uint32_t dim = meta_.dimension();
|
||||
uint32_t type = meta_.data_type();
|
||||
|
||||
switch (type) {
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
float *transposed_tables_ptr =
|
||||
reinterpret_cast<float *>(&transposed_tables_[0]);
|
||||
float *full_pivot_data_ptr =
|
||||
reinterpret_cast<float *>(&full_pivot_data_[0]);
|
||||
for (size_t i = 0; i < kPQCentroidNum; i++) {
|
||||
for (size_t j = 0; j < dim; j++) {
|
||||
transposed_tables_ptr[j * kPQCentroidNum + i] =
|
||||
full_pivot_data_ptr[i * dim + j];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
ailego::Float16 *transposed_tables_ptr =
|
||||
reinterpret_cast<ailego::Float16 *>(&transposed_tables_[0]);
|
||||
ailego::Float16 *full_pivot_data_ptr =
|
||||
reinterpret_cast<ailego::Float16 *>(&full_pivot_data_[0]);
|
||||
for (size_t i = 0; i < kPQCentroidNum; i++) {
|
||||
for (size_t j = 0; j < dim; j++) {
|
||||
transposed_tables_ptr[j * kPQCentroidNum + i] =
|
||||
full_pivot_data_ptr[i * dim + j];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
LOG_ERROR("unsupported type, type: %u", type);
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void PQTable::aggregate_coords(uint32_t id_num, const diskann_id_t *ids,
|
||||
const uint8_t *all_coords, size_t dim,
|
||||
uint8_t *out) {
|
||||
for (size_t i = 0; i < id_num; i++) {
|
||||
memcpy(out + i * dim, all_coords + ids[i] * dim, dim * sizeof(uint8_t));
|
||||
}
|
||||
}
|
||||
|
||||
void PQTable::pq_dist_lookup(const uint8_t *pq_ids, size_t id_num,
|
||||
size_t pq_nchunks, const float *pq_dist_buffer,
|
||||
float *dists_out) {
|
||||
ailego_prefetch(dists_out);
|
||||
ailego_prefetch(pq_ids);
|
||||
ailego_prefetch(pq_ids + 64);
|
||||
ailego_prefetch(pq_ids + 128);
|
||||
|
||||
memset(dists_out, 0, id_num * sizeof(float));
|
||||
|
||||
for (size_t chunk = 0; chunk < pq_nchunks; chunk++) {
|
||||
const float *chunk_dists = pq_dist_buffer + kPQCentroidNum * chunk;
|
||||
if (chunk < pq_nchunks - 1) {
|
||||
ailego_prefetch(chunk_dists + kPQCentroidNum);
|
||||
}
|
||||
for (size_t idx = 0; idx < id_num; idx++) {
|
||||
uint8_t pq_centerid = pq_ids[pq_nchunks * idx + chunk];
|
||||
dists_out[idx] += chunk_dists[pq_centerid];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,154 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class PQTable {
|
||||
public:
|
||||
typedef std::shared_ptr<PQTable> Pointer;
|
||||
|
||||
public:
|
||||
static constexpr uint32_t kPQBitNum = 8;
|
||||
static constexpr uint32_t kPQCentroidNum = 1 << kPQBitNum;
|
||||
static constexpr uint32_t kMaxTrainSampleCount = 200000;
|
||||
static constexpr double kTrainSampleRatio = 1.0;
|
||||
static constexpr uint32_t kMeanIterNum = 12;
|
||||
|
||||
public:
|
||||
PQTable(const IndexMeta &meta, uint32_t chunk_num);
|
||||
virtual ~PQTable();
|
||||
|
||||
int init(std::vector<uint8_t> &table, std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint32_t> &chunk_offsets, std::vector<uint8_t> &pq_data);
|
||||
|
||||
|
||||
template <typename T>
|
||||
void compute_distance_table_ip(const T *query_vec, float *dist_vec) {
|
||||
memset(dist_vec, 0, kPQCentroidNum * chunk_num_ * sizeof(float));
|
||||
|
||||
const T *transposed_tables_ptr =
|
||||
reinterpret_cast<const T *>(transposed_tables_.data());
|
||||
// chunk wise distance computation
|
||||
for (size_t chunk = 0; chunk < chunk_num_; chunk++) {
|
||||
// sum (q-c)^2 for the dimensions associated with this chunk
|
||||
float *chunk_dists = dist_vec + (kPQCentroidNum * chunk);
|
||||
|
||||
for (size_t j = chunk_offsets_[chunk]; j < chunk_offsets_[chunk + 1];
|
||||
j++) {
|
||||
const T *centers_dim_vec = &transposed_tables_ptr[kPQCentroidNum * j];
|
||||
|
||||
for (size_t idx = 0; idx < kPQCentroidNum; idx++) {
|
||||
float centor_data = centers_dim_vec[idx];
|
||||
float query_data = query_vec[j];
|
||||
float dim_score = centor_data * query_data;
|
||||
chunk_dists[idx] += -dim_score;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void compute_distance_table(const T *query_vec, float *dist_vec) {
|
||||
memset(dist_vec, 0, kPQCentroidNum * chunk_num_ * sizeof(float));
|
||||
|
||||
const T *transposed_tables_ptr =
|
||||
reinterpret_cast<const T *>(transposed_tables_.data());
|
||||
// chunk wise distance computation
|
||||
for (size_t chunk = 0; chunk < chunk_num_; chunk++) {
|
||||
// sum (q-c)^2 for the dimensions associated with this chunk
|
||||
float *chunk_dists = dist_vec + (kPQCentroidNum * chunk);
|
||||
|
||||
for (size_t j = chunk_offsets_[chunk]; j < chunk_offsets_[chunk + 1];
|
||||
j++) {
|
||||
const T *centers_dim_vec = &transposed_tables_ptr[kPQCentroidNum * j];
|
||||
|
||||
for (size_t idx = 0; idx < kPQCentroidNum; idx++) {
|
||||
float diff = centers_dim_vec[idx] - query_vec[j];
|
||||
chunk_dists[idx] += (diff * diff);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void preprocess_query(T *query) {
|
||||
const T *centroid_ptr = reinterpret_cast<const T *>(centroid_.data());
|
||||
for (size_t i = 0; i < meta_.dimension(); i++) {
|
||||
query[i] -= centroid_ptr[i];
|
||||
}
|
||||
}
|
||||
|
||||
void aggregate_coords(uint32_t id_num, const diskann_id_t *ids,
|
||||
const uint8_t *all_coords, size_t dim, uint8_t *out);
|
||||
|
||||
void pq_dist_lookup(const uint8_t *pq_ids, size_t id_num, size_t pq_nchunks,
|
||||
const float *pq_dist_buffer, float *dists);
|
||||
|
||||
void compute_dists(uint32_t id_num, const diskann_id_t *ids,
|
||||
uint32_t chunk_num, float *pq_dist_buffer,
|
||||
void *coord_buffer, float *dists) {
|
||||
uint8_t *coord_buffer_ptr = reinterpret_cast<uint8_t *>(coord_buffer);
|
||||
|
||||
aggregate_coords(id_num, ids, this->pq_data(), chunk_num, coord_buffer_ptr);
|
||||
|
||||
pq_dist_lookup(coord_buffer_ptr, id_num, chunk_num, pq_dist_buffer, dists);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
int preprocess_pq_dist_table(void *query_rotated, float *dist_buffer) {
|
||||
switch (meta_.data_type()) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
preprocess_query(reinterpret_cast<float *>(query_rotated));
|
||||
compute_distance_table(reinterpret_cast<float *>(query_rotated),
|
||||
dist_buffer);
|
||||
|
||||
break;
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
preprocess_query(reinterpret_cast<ailego::Float16 *>(query_rotated));
|
||||
compute_distance_table(
|
||||
reinterpret_cast<ailego::Float16 *>(query_rotated), dist_buffer);
|
||||
break;
|
||||
default:
|
||||
LOG_ERROR("Unsupported Type: %u", meta_.data_type());
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
public:
|
||||
const uint8_t *pq_data() const {
|
||||
return pq_data_.data();
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<uint8_t> full_pivot_data_;
|
||||
std::vector<uint8_t> transposed_tables_;
|
||||
|
||||
std::vector<uint8_t> centroid_;
|
||||
std::vector<uint32_t> chunk_offsets_;
|
||||
std::vector<uint8_t> pq_data_;
|
||||
|
||||
IndexMeta meta_;
|
||||
uint64_t chunk_num_{0};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,417 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_pq_trainer.h"
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnPqTrainer::DiskAnnPqTrainer(uint32_t max_train_sample_count)
|
||||
: max_train_sample_count_{max_train_sample_count} {}
|
||||
|
||||
DiskAnnPqTrainer::~DiskAnnPqTrainer() {}
|
||||
|
||||
int DiskAnnPqTrainer::gen_random_sample(IndexHolder::Pointer holder,
|
||||
const IndexMeta &meta,
|
||||
std::string &sample_data,
|
||||
size_t &sample_size) {
|
||||
double train_sample_ratio =
|
||||
max_train_sample_count_ < 1 ? max_train_sample_count_ : 1;
|
||||
|
||||
uint32_t max_train_sample_count = train_sample_ratio * holder->count();
|
||||
max_train_sample_count = max_train_sample_count > max_train_sample_count_
|
||||
? max_train_sample_count_
|
||||
: max_train_sample_count;
|
||||
|
||||
std::vector<std::vector<uint8_t>> sample_vecs;
|
||||
|
||||
// Use a fixed seed for deterministic sampling across runs.
|
||||
uint32_t x = 456321;
|
||||
std::mt19937 gen(x);
|
||||
std::uniform_real_distribution<float> dist(0, 1);
|
||||
|
||||
uint32_t vec_size = meta.element_size();
|
||||
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
size_t sample_count = 0;
|
||||
while (iter->is_valid() && sample_count < max_train_sample_count) {
|
||||
float random = dist(gen);
|
||||
|
||||
if (random < train_sample_ratio) {
|
||||
const void *vec = iter->data();
|
||||
|
||||
std::vector<uint8_t> temp_vec;
|
||||
temp_vec.resize(vec_size);
|
||||
|
||||
std::memcpy(reinterpret_cast<uint8_t *>(&temp_vec[0]), vec, vec_size);
|
||||
|
||||
sample_vecs.push_back(std::move(temp_vec));
|
||||
|
||||
sample_count++;
|
||||
}
|
||||
|
||||
iter->next();
|
||||
}
|
||||
|
||||
sample_size = sample_vecs.size();
|
||||
sample_data.reserve(sample_size * vec_size);
|
||||
|
||||
for (size_t i = 0; i < sample_size; i++) {
|
||||
sample_data.append(reinterpret_cast<const char *>(sample_vecs[i].data()),
|
||||
vec_size);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
int DiskAnnPqTrainer::prepare_pq_train_data(
|
||||
const IndexMeta &meta, size_t num_train, std::string &train_data,
|
||||
bool use_zero_mean, std::vector<uint8_t> ¢roid,
|
||||
std::shared_ptr<CompactIndexFeatures> &train_features) {
|
||||
uint32_t dim = meta.dimension();
|
||||
uint32_t vec_size = meta.element_size();
|
||||
|
||||
std::string train_data_processed;
|
||||
train_data_processed.resize(num_train * vec_size);
|
||||
|
||||
std::memcpy(&(train_data_processed[0]), train_data.data(),
|
||||
num_train * vec_size);
|
||||
|
||||
// use fp32 to accumulate to avoid overflow
|
||||
std::vector<float> centroid_temp(dim);
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
centroid_temp[d] = 0;
|
||||
}
|
||||
|
||||
T *train_data_processed_ptr = reinterpret_cast<T *>(&train_data_processed[0]);
|
||||
|
||||
if (use_zero_mean) {
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
for (uint64_t p = 0; p < num_train; p++) {
|
||||
centroid_temp[d] += train_data_processed_ptr[p * dim + d];
|
||||
}
|
||||
centroid_temp[d] /= num_train;
|
||||
}
|
||||
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
for (uint64_t p = 0; p < num_train; p++) {
|
||||
train_data_processed_ptr[p * dim + d] -= centroid_temp[d];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < num_train; ++i) {
|
||||
train_features->emplace(train_data_processed_ptr + i * dim);
|
||||
}
|
||||
|
||||
// copy the centroid out
|
||||
centroid.resize(vec_size);
|
||||
T *centroid_ptr = reinterpret_cast<T *>(centroid.data());
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
centroid_ptr[d] = centroid_temp[d];
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
int DiskAnnPqTrainer::convert_pivot_data(
|
||||
const IndexMeta &meta, uint32_t num_centers, uint32_t pq_chunk_num,
|
||||
const std::vector<uint32_t> &chunk_dims,
|
||||
const std::vector<uint32_t> &chunk_offsets,
|
||||
IndexCluster::CentroidList ¢roids,
|
||||
std::vector<uint8_t> &full_pivot_data) {
|
||||
uint32_t dim = meta.dimension();
|
||||
uint32_t element_size = meta.element_size();
|
||||
|
||||
full_pivot_data.resize(num_centers * element_size);
|
||||
|
||||
for (size_t chunk = 0; chunk < pq_chunk_num; ++chunk) {
|
||||
for (size_t cluster = 0; cluster < num_centers; ++cluster) {
|
||||
size_t idx = chunk * num_centers + cluster;
|
||||
|
||||
T *pivot_data_ptr = reinterpret_cast<T *>(&(full_pivot_data[0])) +
|
||||
cluster * dim + chunk_offsets[chunk];
|
||||
const T *feature_ptr =
|
||||
reinterpret_cast<const T *>(centroids[idx].feature());
|
||||
for (size_t d = 0; d <= chunk_dims[chunk]; ++d) {
|
||||
pivot_data_ptr[d] = feature_ptr[d];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnPqTrainer::train_pq(IndexThreads::Pointer threads,
|
||||
const IndexMeta &meta, std::string &train_data,
|
||||
size_t num_train, uint32_t num_centers,
|
||||
uint32_t pq_chunk_num, uint32_t max_iterations,
|
||||
bool use_zero_mean,
|
||||
std::vector<uint8_t> &full_pivot_data,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint32_t> &chunk_offsets) {
|
||||
uint32_t dim = meta.dimension();
|
||||
if (pq_chunk_num > dim) {
|
||||
LOG_ERROR("Error: number of chunks more than dimension. chunk: %u, dim: %u",
|
||||
pq_chunk_num, dim);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::shared_ptr<CompactIndexFeatures> train_features(
|
||||
new CompactIndexFeatures(meta));
|
||||
|
||||
uint32_t type = meta.data_type();
|
||||
|
||||
int ret;
|
||||
switch (type) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
ret = prepare_pq_train_data<float>(
|
||||
meta, num_train, train_data, use_zero_mean, centroid, train_features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to prepare pq train data");
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
ret = prepare_pq_train_data<ailego::Float16>(
|
||||
meta, num_train, train_data, use_zero_mean, centroid, train_features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to prepare pq train data");
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Do Train
|
||||
ailego::Params params;
|
||||
params.set(MULTI_CHUNK_CLUSTER_COUNT, num_centers);
|
||||
params.set(MULTI_CHUNK_CLUSTER_CHUNK_COUNT, pq_chunk_num);
|
||||
params.set(MULTI_CHUNK_CLUSTER_MAX_ITERATIONS, max_iterations);
|
||||
|
||||
ret = chunk_cluster_.init(meta, params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to get chunk cluster");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
ret = chunk_cluster_.mount(train_features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Cannot mount train features");
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
std::vector<uint32_t> labels;
|
||||
|
||||
ret = chunk_cluster_.cluster(threads, cluster_centroids_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to cluster");
|
||||
return ret;
|
||||
}
|
||||
|
||||
chunk_offsets = chunk_cluster_.chunk_dim_offsets();
|
||||
auto chunk_dims = chunk_cluster_.chunk_dims();
|
||||
|
||||
switch (type) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
ret = convert_pivot_data<float>(meta, num_centers, pq_chunk_num,
|
||||
chunk_dims, chunk_offsets,
|
||||
cluster_centroids_, full_pivot_data);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to convert pivot data");
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
ret = convert_pivot_data<ailego::Float16>(
|
||||
meta, num_centers, pq_chunk_num, chunk_dims, chunk_offsets,
|
||||
cluster_centroids_, full_pivot_data);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to convert pivot data");
|
||||
return ret;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnPqTrainer::train_quantized_data(
|
||||
IndexThreads::Pointer threads, IndexHolder::Pointer holder,
|
||||
const IndexMeta &meta, std::vector<uint8_t> &pq_full_pivot_data,
|
||||
std::vector<uint8_t> &pq_centroid, std::vector<uint32_t> &pq_chunk_offsets,
|
||||
size_t pq_chunk_num) {
|
||||
size_t train_size;
|
||||
std::string train_data;
|
||||
|
||||
int ret = gen_random_sample(holder, meta, train_data, train_size);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Get Random Sample Error, ret: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("Training data with %zu samples loaded.", train_size);
|
||||
|
||||
// bool use_zero_mean = (meta.metric_name() != "InnerProduct" ? true :
|
||||
// false);
|
||||
bool use_zero_mean = false;
|
||||
|
||||
ret = train_pq(threads, meta, train_data, train_size, PQTable::kPQCentroidNum,
|
||||
pq_chunk_num, PQTable::kMeanIterNum, use_zero_mean,
|
||||
pq_full_pivot_data, pq_centroid, pq_chunk_offsets);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Train PQ Error, ret: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnPqTrainer::generate_pq(IndexThreads::Pointer threads,
|
||||
const IndexMeta &meta,
|
||||
IndexHolder::Pointer holder,
|
||||
uint32_t pq_chunk_num,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint8_t> &block_compressed_data) {
|
||||
uint32_t type = meta.data_type();
|
||||
uint32_t dim = meta.dimension();
|
||||
|
||||
if (pq_chunk_num > dim) {
|
||||
LOG_ERROR("Error: number of chunks more than dimension. chunk: %u, dim: %u",
|
||||
pq_chunk_num, dim);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
// Do Label
|
||||
std::vector<uint32_t> labels;
|
||||
size_t num_vecs = holder->count();
|
||||
size_t batch_size =
|
||||
num_vecs <= compress_batch_size_ ? num_vecs : compress_batch_size_;
|
||||
|
||||
std::vector<uint32_t> block_compressed_base(batch_size * pq_chunk_num);
|
||||
|
||||
std::memset(&block_compressed_base[0], 0,
|
||||
batch_size * pq_chunk_num * sizeof(uint32_t));
|
||||
|
||||
std::vector<uint8_t> block_data(batch_size * meta.element_size());
|
||||
std::vector<uint8_t> block_data_converted(batch_size * meta.element_size());
|
||||
|
||||
size_t block_num = DiskAnnUtil::div_round_up(num_vecs, batch_size);
|
||||
|
||||
block_compressed_data.resize(num_vecs * pq_chunk_num);
|
||||
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
for (size_t block = 0; block < block_num; block++) {
|
||||
size_t start_id = block * batch_size;
|
||||
size_t end_id = std::min((block + 1) * batch_size, num_vecs);
|
||||
|
||||
size_t cur_block_size = end_id - start_id;
|
||||
|
||||
for (size_t i = 0; i < cur_block_size && iter->is_valid(); i++) {
|
||||
const void *vec = iter->data();
|
||||
std::memcpy(
|
||||
reinterpret_cast<uint8_t *>(&block_data[0]) + i * meta.element_size(),
|
||||
vec, meta.element_size());
|
||||
iter->next();
|
||||
}
|
||||
|
||||
std::memcpy(block_data_converted.data(), block_data.data(),
|
||||
cur_block_size * meta.element_size());
|
||||
|
||||
LOG_INFO("Processing Docs, Range: [%zu, %zu)..", start_id, end_id);
|
||||
|
||||
std::shared_ptr<CompactIndexFeatures> block_features(
|
||||
new CompactIndexFeatures(meta));
|
||||
|
||||
switch (type) {
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
DiskAnnUtil::convert_vector_to_residual<float>(
|
||||
reinterpret_cast<float *>(block_data_converted.data()),
|
||||
cur_block_size, dim, centroid.data());
|
||||
break;
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
DiskAnnUtil::convert_vector_to_residual<ailego::Float16>(
|
||||
reinterpret_cast<ailego::Float16 *>(block_data_converted.data()),
|
||||
cur_block_size, dim, centroid.data());
|
||||
break;
|
||||
default:
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < cur_block_size; i++) {
|
||||
block_features->emplace(block_data_converted.data() +
|
||||
i * meta.element_size());
|
||||
}
|
||||
|
||||
int ret = chunk_cluster_.mount(block_features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Cannot mount block features");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = chunk_cluster_.label(threads, cluster_centroids_, &labels);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to label");
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> compressed_data(cur_block_size * pq_chunk_num);
|
||||
|
||||
DiskAnnUtil::convert_types_uint32_to_uint8(
|
||||
labels.data(), compressed_data.data(), cur_block_size, pq_chunk_num);
|
||||
|
||||
memcpy(&(block_compressed_data[0]) + start_id * pq_chunk_num,
|
||||
compressed_data.data(), cur_block_size * pq_chunk_num);
|
||||
|
||||
LOG_INFO("Generate PQ Data Done.");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnPqTrainer::generate_quantized_data(
|
||||
IndexThreads::Pointer threads, IndexHolder::Pointer holder,
|
||||
const IndexMeta &meta, std::vector<uint8_t> &pq_centroid,
|
||||
std::vector<uint8_t> &block_compressed_data, size_t pq_chunk_num) {
|
||||
// bool use_zero_mean = (meta.metric_name() != "InnerProduct" ? true :
|
||||
// false);
|
||||
|
||||
int ret = generate_pq(threads, meta, holder, pq_chunk_num, pq_centroid,
|
||||
block_compressed_data);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Generate PQ Error, ret: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_pq_table.h"
|
||||
#include "../cluster/multi_chunk_cluster.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnPqTrainer {
|
||||
public:
|
||||
typedef std::unique_ptr<DiskAnnPqTrainer> UPointer;
|
||||
|
||||
public:
|
||||
DiskAnnPqTrainer(uint32_t max_train_sample_count);
|
||||
virtual ~DiskAnnPqTrainer();
|
||||
|
||||
public:
|
||||
template <typename T>
|
||||
static int prepare_pq_train_data(
|
||||
const IndexMeta &meta, size_t num_train, std::string &train_data,
|
||||
bool use_zero_mean, std::vector<uint8_t> ¢roid,
|
||||
std::shared_ptr<CompactIndexFeatures> &train_features);
|
||||
|
||||
template <typename T>
|
||||
static int convert_pivot_data(const IndexMeta &meta, uint32_t num_centers,
|
||||
uint32_t pq_chunk_num,
|
||||
const std::vector<uint32_t> &chunk_dims,
|
||||
const std::vector<uint32_t> &chunk_offsets,
|
||||
IndexCluster::CentroidList ¢roids,
|
||||
std::vector<uint8_t> &full_pivot_data);
|
||||
|
||||
int gen_random_sample(IndexHolder::Pointer holder, const IndexMeta &meta,
|
||||
std::string &sample_data, size_t &sample_size);
|
||||
|
||||
int generate_quantized_data(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder,
|
||||
const IndexMeta &meta,
|
||||
std::vector<uint8_t> &pq_centroid,
|
||||
std::vector<uint8_t> &block_compressed_data,
|
||||
size_t num_pq_chunks);
|
||||
|
||||
int generate_pq(IndexThreads::Pointer threads, const IndexMeta &meta,
|
||||
IndexHolder::Pointer holder, uint32_t num_pq_chunks,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint8_t> &block_compressed_data);
|
||||
|
||||
int train_quantized_data(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder, const IndexMeta &meta,
|
||||
std::vector<uint8_t> &pq_full_pivot_data,
|
||||
std::vector<uint8_t> &pq_centroid,
|
||||
std::vector<uint32_t> &pq_chunk_offsets,
|
||||
size_t num_pq_chunks);
|
||||
|
||||
int train_pq(IndexThreads::Pointer threads, const IndexMeta &meta,
|
||||
std::string &train_data, size_t num_train, uint32_t num_centers,
|
||||
uint32_t num_pq_chunks, uint32_t max_iterations,
|
||||
bool use_zero_mean, std::vector<uint8_t> &full_pivot_data,
|
||||
std::vector<uint8_t> ¢roid,
|
||||
std::vector<uint32_t> &chunk_offsets);
|
||||
|
||||
private:
|
||||
static constexpr uint32_t compress_batch_size_{
|
||||
DiskAnnEntity::kDefaultCompressBatchSize};
|
||||
|
||||
// pq cluster
|
||||
MultiChunkCluster chunk_cluster_;
|
||||
IndexCluster::CentroidList cluster_centroids_;
|
||||
uint32_t max_train_sample_count_{PQTable::kMaxTrainSampleCount};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,190 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_reducer.h"
|
||||
#include <vector>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <ailego/parallel/multi_thread_list.h>
|
||||
#include <zvec/ailego/utility/string_helper.h>
|
||||
#include <zvec/core/framework/index_reducer.h>
|
||||
#include "diskann_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int DiskAnnReducer::init(const ailego::Params ¶ms) {
|
||||
params.get(PARAM_DISKANN_REDUCER_WORKING_PATH, &working_path_);
|
||||
if (working_path_.empty()) {
|
||||
LOG_ERROR("Missing parameter. %s",
|
||||
PARAM_DISKANN_REDUCER_WORKING_PATH.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::string index_name =
|
||||
params.get_as_string(PARAM_DISKANN_REDUCER_INDEX_NAME);
|
||||
if (index_name.empty()) {
|
||||
index_name = std::to_string(std::clock());
|
||||
}
|
||||
|
||||
reducer_file_path_ = ailego::StringHelper::Concat(
|
||||
working_path_, "/", kReducerFileName, index_name);
|
||||
|
||||
holder_file_path_ = ailego::StringHelper::Concat(working_path_, "/",
|
||||
kHolderFileName, index_name);
|
||||
|
||||
state_ = STATE_INITED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducer::cleanup(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Reduce operator with filter
|
||||
int DiskAnnReducer::reduce(const IndexFilter &filter) {
|
||||
if (entities_.empty() || state_ != STATE_FEED) {
|
||||
LOG_ERROR("No container to reduce, feed first");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (use_mem_holder_) {
|
||||
mem_holder_ = std::make_shared<RandomAccessIndexHolder>(meta_);
|
||||
for (auto entity : entities_) {
|
||||
size_t doc_cnt = entity->doc_cnt();
|
||||
for (size_t id = 0; id < doc_cnt; ++id) {
|
||||
diskann_key_t pkey = entity->get_key(id);
|
||||
|
||||
if (filter.is_valid() && filter(pkey)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const void *vec = entity->get_vector(id);
|
||||
mem_holder_->emplace(pkey, vec);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
disk_holder_ =
|
||||
std::make_shared<DiskAnnIndexHolder>(meta_, holder_file_path_);
|
||||
|
||||
int ret = disk_holder_->init();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("DiskAnn Index Holder init failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
for (auto entity : entities_) {
|
||||
size_t doc_cnt = entity->doc_cnt();
|
||||
for (size_t id = 0; id < doc_cnt; ++id) {
|
||||
diskann_key_t pkey = entity->get_key(id);
|
||||
|
||||
if (filter.is_valid() && filter(pkey)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const void *vec = entity->get_vector(id);
|
||||
disk_holder_->emplace(pkey, vec);
|
||||
}
|
||||
}
|
||||
|
||||
disk_holder_->close();
|
||||
}
|
||||
|
||||
builder_ = IndexFactory::CreateBuilder(kDiskAnnBuilderName);
|
||||
if (!builder_) {
|
||||
LOG_ERROR("Create builder failed. name[%s]", kDiskAnnBuilderName.c_str());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
if (thread_pool_ == nullptr) {
|
||||
LOG_ERROR(
|
||||
"Only support multi-thread mode. Thread pool is not set for reducer.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Start diskann reduce");
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
|
||||
auto params = meta_.builder_params();
|
||||
|
||||
int ret = builder_->init(meta_, params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Init proxima streamer failed. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (use_mem_holder_) {
|
||||
ret = builder_->train(mem_holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Diskann builder failed to train. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = builder_->build(mem_holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Diskann builder failed to build. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
ret = builder_->train(disk_holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Diskann builder failed to train. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = builder_->build(disk_holder_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Diskann builder failed to build. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
auto &stats = builder_->stats();
|
||||
|
||||
stats_.set_reduced_costtime(timer.seconds());
|
||||
stats_.set_filtered_count(stats.discarded_count());
|
||||
|
||||
state_ = STATE_REDUCE;
|
||||
|
||||
LOG_INFO("End DiskAnn reduce. cost time: [%zu]s", (size_t)timer.seconds());
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Dump index by dumper
|
||||
int DiskAnnReducer::dump(const IndexDumper::Pointer &dumper) {
|
||||
LOG_INFO("Begin diskann reducer dump");
|
||||
|
||||
if (state_ != STATE_REDUCE) {
|
||||
LOG_WARN("Reduce first before dump.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
int ret = builder_->dump(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("diskann reducer dump failed. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
stats_.set_dumped_costtime(timer.seconds());
|
||||
|
||||
LOG_INFO("End diskann reducer dump, dump costtime=[%zu]s",
|
||||
(size_t)(stats_.dumped_costtime()));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_REDUCER(DiskAnnReducer);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <ailego/parallel/multi_thread_list.h>
|
||||
#include <utility/sparse_utility.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include <zvec/core/framework/index_reducer.h>
|
||||
#include "diskann_holder.h"
|
||||
#include "diskann_reducer_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnReducer : public IndexReducer {
|
||||
public:
|
||||
//! Constructor
|
||||
DiskAnnReducer(void) = default;
|
||||
|
||||
protected:
|
||||
//! Initialize Reducer
|
||||
virtual int init(const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Reducer
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Feed indexes from containers
|
||||
// virtual int feed(IndexStorage::Pointer container) override;
|
||||
|
||||
//! Reduce operator (with filter)
|
||||
virtual int reduce(const IndexFilter &filter) override;
|
||||
|
||||
//! Dump index by dumper
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
private:
|
||||
enum State {
|
||||
STATE_UNINITED = 0,
|
||||
STATE_INITED = 1,
|
||||
STATE_FEED = 2,
|
||||
STATE_REDUCE = 3
|
||||
};
|
||||
|
||||
std::string working_path_{""};
|
||||
|
||||
IndexMeta meta_{};
|
||||
std::vector<DiskAnnReducerEntity::Pointer> entities_{};
|
||||
|
||||
// bool use_mem_holder_{true};
|
||||
bool use_mem_holder_{false};
|
||||
RandomAccessIndexHolder::Pointer mem_holder_;
|
||||
DiskAnnIndexHolder::Pointer disk_holder_;
|
||||
|
||||
IndexBuilder::Pointer builder_{nullptr};
|
||||
std::string reducer_file_path_{""};
|
||||
std::string holder_file_path_{""};
|
||||
|
||||
Stats stats_{};
|
||||
State state_{STATE_UNINITED};
|
||||
|
||||
const std::string kDiskAnnBuilderName{"DiskAnnBuilder"};
|
||||
const std::string kReducerFileName{"diskann.reducer.builder."};
|
||||
const std::string kHolderFileName{"diskann.reducer.holder."};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,215 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_reducer_entity.h"
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int DiskAnnReducerEntity::load(const IndexStorage::Pointer &container,
|
||||
bool check_crc) {
|
||||
container_ = container;
|
||||
|
||||
int ret = load_segments(check_crc);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
sector_num_per_node_ = node_per_sector() > 0
|
||||
? 1
|
||||
: DiskAnnUtil::div_round_up(
|
||||
max_node_size(), DiskAnnUtil::kSectorSize);
|
||||
|
||||
loaded_ = true;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducerEntity::load_segments(bool /*check_crc*/) {
|
||||
int ret;
|
||||
ret = load_header_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Header Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_key_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Key Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_vector_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Vector Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducerEntity::load_header_segment() {
|
||||
const void *data = nullptr;
|
||||
meta_segment_ = container_->get(kDiskAnnMetaSegmentId);
|
||||
if (!meta_segment_ ||
|
||||
meta_segment_->data_size() < sizeof(DiskAnnMetaHeader)) {
|
||||
LOG_ERROR("Miss or invalid segment %s", kDiskAnnMetaSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (meta_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
sizeof(DiskAnnMetaHeader)) !=
|
||||
sizeof(DiskAnnMetaHeader)) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnMetaSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
::memcpy(reinterpret_cast<uint8_t *>(&meta_header_), data,
|
||||
sizeof(DiskAnnMetaHeader));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducerEntity::load_vector_segment() {
|
||||
vector_segment_ = container_->get(kDiskAnnVectorSegmentId);
|
||||
if (!vector_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnVectorSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnReducerEntity::load_key_segment() {
|
||||
// load key
|
||||
key_segment_ = container_->get(kDiskAnnKeySegmentId);
|
||||
if (!key_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnKeySegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
size_t key_data_len = doc_cnt() * sizeof(key_t);
|
||||
|
||||
// load key mapping
|
||||
key_mapping_segment_ = container_->get(kDiskAnnKeyMappingSegmentId);
|
||||
const void *data = nullptr;
|
||||
if (key_mapping_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
key_data_len) != key_data_len) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnKeyMappingSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
key_buffer_.resize(key_data_len);
|
||||
memcpy(&(key_buffer_[0]), data, key_data_len);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool DiskAnnReducerEntity::do_crc_check(
|
||||
std::vector<SegmentPointer> &segments) const {
|
||||
constexpr size_t blk_size = 4096;
|
||||
const void *data;
|
||||
|
||||
for (auto &segment : segments) {
|
||||
size_t offset = 0;
|
||||
size_t rd_size;
|
||||
uint32_t crc = 0;
|
||||
while (offset < segment->data_size()) {
|
||||
size_t size = std::min(blk_size, segment->data_size() - offset);
|
||||
if ((rd_size = segment->read(offset, &data, size)) <= 0) {
|
||||
break;
|
||||
}
|
||||
offset += rd_size;
|
||||
crc = ailego::Crc32c::Hash(data, rd_size, crc);
|
||||
}
|
||||
if (crc != segment->data_crc()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Get vector local id by key
|
||||
diskann_id_t DiskAnnReducerEntity::get_id(diskann_key_t key) const {
|
||||
const diskann_id_t *key_mapping_data_ptr =
|
||||
reinterpret_cast<const diskann_id_t *>(key_mapping_buffer_.data());
|
||||
const diskann_key_t *key_data_ptr =
|
||||
reinterpret_cast<const diskann_key_t *>(key_buffer_.data());
|
||||
|
||||
//! Do binary search
|
||||
diskann_id_t start = 0UL;
|
||||
diskann_id_t end = doc_cnt();
|
||||
diskann_id_t idx = 0u;
|
||||
while (start < end) {
|
||||
idx = start + (end - start) / 2;
|
||||
diskann_id_t local_id = key_mapping_data_ptr[idx];
|
||||
|
||||
const diskann_key_t local_key = key_data_ptr[local_id];
|
||||
|
||||
if (local_key < key) {
|
||||
start = idx + 1;
|
||||
} else if (local_key > key) {
|
||||
end = idx;
|
||||
} else {
|
||||
return local_id;
|
||||
}
|
||||
}
|
||||
|
||||
return kInvalidId;
|
||||
}
|
||||
|
||||
diskann_key_t DiskAnnReducerEntity::get_key(diskann_id_t id) const {
|
||||
const void *key;
|
||||
if (ailego_unlikely(key_segment_->read(id * sizeof(diskann_key_t), &key,
|
||||
sizeof(diskann_key_t)) !=
|
||||
sizeof(diskann_key_t))) {
|
||||
LOG_ERROR("Read key from segment failed");
|
||||
return kInvalidKey;
|
||||
}
|
||||
|
||||
return *(reinterpret_cast<const diskann_key_t *>(key));
|
||||
}
|
||||
|
||||
const void *DiskAnnReducerEntity::get_vector(diskann_id_t id) const {
|
||||
size_t read_size = sector_num_per_node_ * DiskAnnUtil::kSectorSize;
|
||||
size_t sector_id = DiskAnnUtil::get_node_sector(
|
||||
node_per_sector(), max_node_size(), DiskAnnUtil::kSectorSize, id);
|
||||
size_t offset = sector_id * DiskAnnUtil::kSectorSize;
|
||||
|
||||
if (sector_id != sector_id_) {
|
||||
const void *sector_data;
|
||||
if (ailego_unlikely(vector_segment_->read(offset, §or_data,
|
||||
read_size) != read_size)) {
|
||||
LOG_ERROR("Read vector from segment failed");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
sector_id_ = sector_id;
|
||||
sector_buffer_.assign(reinterpret_cast<const char *>(sector_data),
|
||||
read_size);
|
||||
}
|
||||
|
||||
return DiskAnnUtil::offset_to_node_const(
|
||||
node_per_sector(), max_node_size(),
|
||||
reinterpret_cast<const uint8_t *>(sector_buffer_.data()), id);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_file_reader.h"
|
||||
#include "diskann_pq_table.h"
|
||||
#include "diskann_util.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnReducerEntity : public DiskAnnEntity {
|
||||
public:
|
||||
using Pointer = std::shared_ptr<DiskAnnReducerEntity>;
|
||||
using SegmentPointer = IndexStorage::Segment::Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnReducerEntity() = default;
|
||||
virtual ~DiskAnnReducerEntity() = default;
|
||||
|
||||
int load(const IndexStorage::Pointer &container, bool check_crc);
|
||||
int load_segments(bool check_crc);
|
||||
int load_header_segment();
|
||||
int load_vector_segment();
|
||||
int load_key_segment();
|
||||
int load_key_mapping_segment();
|
||||
|
||||
bool do_crc_check(std::vector<SegmentPointer> &segments) const;
|
||||
|
||||
diskann_id_t get_id(diskann_key_t key) const override;
|
||||
diskann_key_t get_key(diskann_id_t id) const override;
|
||||
const void *get_vector(diskann_id_t id) const override;
|
||||
|
||||
private:
|
||||
IndexStorage::Pointer container_{};
|
||||
IndexStorage::Segment::Pointer meta_segment_{};
|
||||
IndexStorage::Segment::Pointer vector_segment_{};
|
||||
IndexStorage::Segment::Pointer key_segment_{};
|
||||
IndexStorage::Segment::Pointer key_mapping_segment_{};
|
||||
|
||||
std::string key_buffer_;
|
||||
std::string key_mapping_buffer_;
|
||||
|
||||
size_t sector_num_per_node_{0};
|
||||
|
||||
mutable size_t sector_id_{-1U};
|
||||
mutable std::string sector_buffer_;
|
||||
|
||||
bool loaded_{false};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,312 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_searcher.h"
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_indexer.h"
|
||||
#include "diskann_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnSearcher::DiskAnnSearcher() {}
|
||||
|
||||
DiskAnnSearcher::~DiskAnnSearcher() {}
|
||||
|
||||
int DiskAnnSearcher::init(const ailego::Params &search_params) {
|
||||
search_params.get(PARAM_DISKANN_SEARCHER_LIST_SIZE, &list_size_);
|
||||
search_params.get(PARAM_DISKANN_SEARCHER_CACHE_NODE_NUM, &cache_nodes_num_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void DiskAnnSearcher::print_debug_info() {}
|
||||
|
||||
int DiskAnnSearcher::cleanup() {
|
||||
LOG_INFO("Begin DiskAnnSearcher:cleanup");
|
||||
|
||||
LOG_INFO("End DiskAnnSearcher:cleanup");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::load(IndexStorage::Pointer storage,
|
||||
IndexMetric::Pointer measure) {
|
||||
LOG_INFO("DiskAnnSearcher::load Begin");
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
int ret = IndexHelper::DeserializeFromStorage(storage.get(), &meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to deserialize meta from storage");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = entity_.load(meta_, storage);
|
||||
if (ret != 0) {
|
||||
LOG_INFO("Searcher Entity Load Failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
diskann_indexer_ = std::make_shared<DiskAnnIndexer>(meta_);
|
||||
|
||||
int res = diskann_indexer_->init(entity_);
|
||||
if (res != 0) {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (cache_nodes_num_ != 0) {
|
||||
std::vector<diskann_id_t> node_list;
|
||||
LOG_INFO("Caching %u nodes around medoid(s)", cache_nodes_num_);
|
||||
|
||||
diskann_indexer_->cache_bfs_levels(cache_nodes_num_, node_list);
|
||||
|
||||
diskann_indexer_->load_cache_list(node_list);
|
||||
|
||||
node_list.clear();
|
||||
node_list.shrink_to_fit();
|
||||
}
|
||||
|
||||
if (measure) {
|
||||
measure_ = measure;
|
||||
} else {
|
||||
measure_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!measure_) {
|
||||
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
ret = measure_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
if (measure_->query_metric()) {
|
||||
measure_ = measure_->query_metric();
|
||||
}
|
||||
}
|
||||
|
||||
stats_.set_loaded_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
state_ = STATE_LOADED;
|
||||
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
LOG_INFO("DiskAnnSearcher::load Done");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::unload() {
|
||||
LOG_INFO("DiskAnnSearcher unload index");
|
||||
|
||||
state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::update_context(DiskAnnContext *ctx) const {
|
||||
const DiskAnnEntity::Pointer entity = entity_.clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone search context entity");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return ctx->update_context(DiskAnnContext::kSearcherContext, meta_, measure_,
|
||||
entity, magic_);
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
// do search
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
// Context is pooled per index type. When switching between DiskAnn indexes
|
||||
// with different element sizes (e.g., fp16 vs fp32), the cached context has
|
||||
// undersized buffers. Recreate it to ensure correct buffer allocations.
|
||||
if (ctx->magic() != magic_) {
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->knn_search(ctx);
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::search_bf_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer, recreate it
|
||||
//! to ensure buffers are correctly sized for this index's parameters.
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->linear_search(ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(p_keys.size() != count)) {
|
||||
LOG_ERROR("The size of p_keys is not equal to count");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer, recreate it
|
||||
//! to ensure buffers are correctly sized for this index's parameters.
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->keys_search(p_keys[i], ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcher::get_vector(uint64_t key, Context::Pointer &context,
|
||||
std::string &vector) const {
|
||||
return diskann_indexer_->get_vector(key, context, vector);
|
||||
}
|
||||
|
||||
IndexSearcher::Context::Pointer DiskAnnSearcher::create_context() const {
|
||||
const DiskAnnEntity::Pointer search_ctx_entity = entity_.clone();
|
||||
if (!search_ctx_entity) {
|
||||
LOG_ERROR("Failed to create search context entity");
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx =
|
||||
new (std::nothrow) DiskAnnContext(meta_, measure_, search_ctx_entity);
|
||||
if (ctx == nullptr) {
|
||||
LOG_ERROR("Failed to allocate DiskAnn Context");
|
||||
return Context::Pointer();
|
||||
}
|
||||
if (ailego_unlikely(ctx->init(
|
||||
DiskAnnContext::kSearcherContext, search_ctx_entity->max_degree(),
|
||||
search_ctx_entity->pq_chunk_num(), meta_.element_size())) != 0) {
|
||||
LOG_ERROR("Init DiskAnn Context failed");
|
||||
delete ctx;
|
||||
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
ctx->set_list_size(list_size_);
|
||||
ctx->set_magic(magic_);
|
||||
|
||||
return Context::Pointer(ctx);
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_SEARCHER(DiskAnnSearcher);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,166 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_indexer.h"
|
||||
|
||||
class LinuxAlignedFileReader;
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnSearcher : public IndexSearcher {
|
||||
public:
|
||||
using ContextPointer = IndexSearcher::Context::Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnSearcher(void);
|
||||
~DiskAnnSearcher(void);
|
||||
|
||||
DiskAnnSearcher(const DiskAnnSearcher &) = delete;
|
||||
DiskAnnSearcher &operator=(const DiskAnnSearcher &) = delete;
|
||||
|
||||
protected:
|
||||
//! Initialize Searcher
|
||||
int init(const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Searcher
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Load Index from storage
|
||||
int load(IndexStorage::Pointer storage, IndexMetric::Pointer metric) override;
|
||||
|
||||
//! Unload index from storage
|
||||
int unload(void) override;
|
||||
|
||||
//! KNN Search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! KNN Search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, ContextPointer &context) const override;
|
||||
|
||||
//! Linear Search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear Search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, ContextPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query, const uint32_t sparse_count,
|
||||
const uint32_t *sparse_indices,
|
||||
const void *sparse_query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, &sparse_count, sparse_indices,
|
||||
sparse_query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(
|
||||
const void * /*query*/, const uint32_t * /*sparse_count*/,
|
||||
const uint32_t * /*sparse_indices*/, const void * /*sparse_query*/,
|
||||
const std::vector<std::vector<uint64_t>> & /*p_keys*/,
|
||||
const IndexQueryMeta & /*qmeta*/, uint32_t /*count*/,
|
||||
ContextPointer & /*context*/) const override {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Get vector by key
|
||||
int get_vector(uint64_t key, Context::Pointer &context,
|
||||
std::string &vector) const override;
|
||||
|
||||
//! Create a searcher context
|
||||
ContextPointer create_context() const override;
|
||||
|
||||
//! Create a new iterator
|
||||
IndexSearcher::Provider::Pointer create_provider(void) const override {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//! Retrieve statistics
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
//! Retrieve params of index
|
||||
const ailego::Params ¶ms(void) const override {
|
||||
return params_;
|
||||
}
|
||||
|
||||
void print_debug_info() override;
|
||||
|
||||
private:
|
||||
template <typename T, typename LabelT = uint32_t>
|
||||
int search_disk_index(const std::string &query_file,
|
||||
const uint32_t num_nodes_to_cache,
|
||||
const uint32_t recall_at, const uint32_t beamwidth);
|
||||
|
||||
//! To share ctx across streamer/searcher, we need to update the context for
|
||||
//! current streamer/searcher
|
||||
int update_context(DiskAnnContext *ctx) const;
|
||||
|
||||
private:
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
|
||||
|
||||
IndexMetric::Pointer measure_{};
|
||||
IndexMeta meta_{};
|
||||
ailego::Params params_{};
|
||||
|
||||
uint32_t list_size_{200};
|
||||
uint32_t cache_nodes_num_{0};
|
||||
|
||||
bool warm_up_{false};
|
||||
uint32_t beam_size_{2};
|
||||
|
||||
DiskAnnIndexer::Pointer diskann_indexer_{nullptr};
|
||||
DiskAnnSearcherEntity entity_{};
|
||||
|
||||
uint32_t magic_{0U};
|
||||
|
||||
Stats stats_;
|
||||
State state_{STATE_INIT};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,441 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_searcher_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
const DiskAnnEntity::Pointer DiskAnnSearcherEntity::clone() const {
|
||||
auto meta_segment = meta_segment_->clone();
|
||||
if (ailego_unlikely(!meta_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnMetaSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto pq_meta_segment = pq_meta_segment_->clone();
|
||||
if (ailego_unlikely(!pq_meta_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnPqMetaSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto pq_data_segment = pq_data_segment_->clone();
|
||||
if (ailego_unlikely(!pq_data_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnPqDataSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto vector_segment = vector_segment_->clone();
|
||||
if (ailego_unlikely(!vector_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnVectorSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto key_segment = key_segment_->clone();
|
||||
if (ailego_unlikely(!key_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnKeySegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto key_mapping_segment = key_mapping_segment_->clone();
|
||||
if (ailego_unlikely(!key_mapping_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnKeyMappingSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
auto entrypoint_segment = entrypoint_segment_->clone();
|
||||
if (ailego_unlikely(!entrypoint_segment)) {
|
||||
LOG_ERROR("clone segment %s failed", kDiskAnnEntryPointSegmentId.c_str());
|
||||
return DiskAnnEntity::Pointer();
|
||||
}
|
||||
|
||||
DiskAnnSearcherEntity *entity = new (std::nothrow) DiskAnnSearcherEntity(
|
||||
meta_header_, pq_meta_, meta_segment, pq_meta_segment, pq_data_segment,
|
||||
vector_segment, key_segment, key_mapping_segment, entrypoint_segment,
|
||||
num_threads_, list_size_, cache_nodes_num_, warm_up_, beam_size_, meta_,
|
||||
pq_table_, key_buffer_, key_mapping_buffer_, entrypoints_);
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("DiskAnnSearcherEntity new failed");
|
||||
}
|
||||
|
||||
return DiskAnnEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load(const IndexMeta &meta,
|
||||
IndexStorage::Pointer storage) {
|
||||
meta_ = meta;
|
||||
|
||||
storage_ = storage;
|
||||
|
||||
int ret;
|
||||
ret = load_header_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Header Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_pq_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load PQ Meta Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_key_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Key Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_key_mapping_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Key Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_entrypoint_segment();
|
||||
if (ret != 0) {
|
||||
LOG_WARN("Load EntryPoint Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = load_vector_segment();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Load Vector Segment Failed, ret = %d", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_pq_segment() {
|
||||
const void *data = nullptr;
|
||||
|
||||
// load pq meta
|
||||
pq_meta_segment_ = storage_->get(DiskAnnEntity::kDiskAnnPqMetaSegmentId);
|
||||
if (!pq_meta_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
size_t read_size;
|
||||
size_t offset = 0;
|
||||
|
||||
// 1. read pq meta
|
||||
read_size = pq_meta_segment_->read(offset, &data, sizeof(DiskAnnPqMeta));
|
||||
if (read_size != sizeof(DiskAnnPqMeta)) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
sizeof(DiskAnnPqMeta), read_size);
|
||||
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
memcpy(reinterpret_cast<uint8_t *>(&pq_meta_), data, sizeof(DiskAnnPqMeta));
|
||||
offset += read_size;
|
||||
|
||||
// 2. read full pivot data
|
||||
std::vector<uint8_t> full_pivot_data;
|
||||
full_pivot_data.resize(pq_meta_.full_pivot_data_size);
|
||||
|
||||
read_size =
|
||||
pq_meta_segment_->read(offset, &data, pq_meta_.full_pivot_data_size);
|
||||
if (read_size != pq_meta_.full_pivot_data_size) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
(size_t)(pq_meta_.full_pivot_data_size), (size_t)read_size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(&(full_pivot_data[0]), data, read_size);
|
||||
offset += read_size;
|
||||
|
||||
// 3. read centroid
|
||||
std::vector<uint8_t> centroid;
|
||||
centroid.resize(pq_meta_.centroid_data_size);
|
||||
|
||||
read_size =
|
||||
pq_meta_segment_->read(offset, &data, pq_meta_.centroid_data_size);
|
||||
if (read_size != pq_meta_.centroid_data_size) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
(size_t)(pq_meta_.centroid_data_size), (size_t)read_size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(&(centroid[0]), data, read_size);
|
||||
offset += read_size;
|
||||
|
||||
// 4. chunk offset
|
||||
std::vector<uint32_t> chunk_offsets;
|
||||
chunk_offsets.resize(pq_meta_.chunk_num + 1);
|
||||
|
||||
read_size = pq_meta_segment_->read(
|
||||
offset, &data, (pq_meta_.chunk_num + 1) * sizeof(uint32_t));
|
||||
if (read_size != (pq_meta_.chunk_num + 1) * sizeof(uint32_t)) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
(size_t)((pq_meta_.chunk_num + 1) * sizeof(uint32_t)),
|
||||
(size_t)read_size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(&(chunk_offsets[0]), data, read_size);
|
||||
|
||||
// load pq data
|
||||
std::vector<uint8_t> pq_data;
|
||||
pq_data_segment_ = storage_->get(DiskAnnEntity::kDiskAnnPqDataSegmentId);
|
||||
if (!pq_data_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnPqDataSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
pq_data.resize(meta_header_.doc_cnt * pq_meta_.chunk_num);
|
||||
|
||||
void *pq_data_ptr = &pq_data[0];
|
||||
read_size = pq_data_segment_->fetch(
|
||||
0, pq_data_ptr, meta_header_.doc_cnt * pq_meta_.chunk_num);
|
||||
|
||||
if (read_size != meta_header_.doc_cnt * pq_meta_.chunk_num) {
|
||||
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
|
||||
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
|
||||
(size_t)(meta_header_.doc_cnt * pq_meta_.chunk_num),
|
||||
(size_t)read_size);
|
||||
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
pq_table_ = std::make_shared<PQTable>(meta_, pq_meta_.chunk_num);
|
||||
|
||||
pq_table_->init(full_pivot_data, centroid, chunk_offsets, pq_data);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_header_segment() {
|
||||
const void *data = nullptr;
|
||||
meta_segment_ = storage_->get(kDiskAnnMetaSegmentId);
|
||||
if (!meta_segment_ ||
|
||||
meta_segment_->data_size() < sizeof(DiskAnnMetaHeader)) {
|
||||
LOG_ERROR("Miss or invalid segment %s", kDiskAnnMetaSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (meta_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
sizeof(DiskAnnMetaHeader)) !=
|
||||
sizeof(DiskAnnMetaHeader)) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnMetaSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(reinterpret_cast<uint8_t *>(&meta_header_), data,
|
||||
sizeof(DiskAnnMetaHeader));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_vector_segment() {
|
||||
vector_segment_ = storage_->get(kDiskAnnVectorSegmentId);
|
||||
if (!vector_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnVectorSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_key_segment() {
|
||||
// load key
|
||||
key_segment_ = storage_->get(kDiskAnnKeySegmentId);
|
||||
if (!key_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnKeySegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
size_t key_data_len = doc_cnt() * sizeof(diskann_key_t);
|
||||
|
||||
const void *data = nullptr;
|
||||
if (key_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
key_data_len) != key_data_len) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnKeySegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
key_buffer_.resize(key_data_len);
|
||||
memcpy(&(key_buffer_[0]), data, key_data_len);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnSearcherEntity::load_entrypoint_segment() {
|
||||
entrypoint_segment_ = storage_->get(kDiskAnnEntryPointSegmentId);
|
||||
if (!entrypoint_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnEntryPointSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
const void *data = nullptr;
|
||||
|
||||
if (entrypoint_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
sizeof(uint32_t)) != sizeof(uint32_t)) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnEntryPointSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
uint32_t entrypoint_cnt = 0;
|
||||
memcpy(&entrypoint_cnt, data, sizeof(uint32_t));
|
||||
|
||||
if (entrypoint_cnt != 0) {
|
||||
size_t entrypoint_data_len = entrypoint_cnt * sizeof(diskann_id_t);
|
||||
|
||||
if (entrypoint_segment_->read(sizeof(uint32_t),
|
||||
reinterpret_cast<const void **>(&data),
|
||||
entrypoint_data_len) != entrypoint_data_len) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnEntryPointSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
entrypoints_.resize(entrypoint_cnt);
|
||||
memcpy(&(entrypoints_[0]), data, entrypoint_data_len);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int DiskAnnSearcherEntity::load_key_mapping_segment() {
|
||||
key_mapping_segment_ = storage_->get(kDiskAnnKeyMappingSegmentId);
|
||||
if (!key_mapping_segment_) {
|
||||
LOG_ERROR("Miss or invalid segment %s",
|
||||
DiskAnnEntity::kDiskAnnKeyMappingSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
size_t key_mapping_data_len = doc_cnt() * sizeof(diskann_id_t);
|
||||
|
||||
const void *data = nullptr;
|
||||
if (key_mapping_segment_->read(0, reinterpret_cast<const void **>(&data),
|
||||
key_mapping_data_len) !=
|
||||
key_mapping_data_len) {
|
||||
LOG_ERROR("Read segment %s failed", kDiskAnnKeyMappingSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
key_mapping_buffer_.resize(key_mapping_data_len);
|
||||
memcpy(&(key_mapping_buffer_[0]), data, key_mapping_data_len);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Get vector local id by key
|
||||
diskann_id_t DiskAnnSearcherEntity::get_id(diskann_key_t key) const {
|
||||
const diskann_id_t *key_mapping_data_ptr =
|
||||
reinterpret_cast<const diskann_id_t *>(key_mapping_buffer_.data());
|
||||
|
||||
const diskann_key_t *key_data_ptr =
|
||||
reinterpret_cast<const diskann_key_t *>(key_buffer_.data());
|
||||
|
||||
//! Do binary search
|
||||
diskann_id_t start = 0UL;
|
||||
diskann_id_t end = doc_cnt();
|
||||
diskann_id_t idx = 0u;
|
||||
while (start < end) {
|
||||
idx = start + (end - start) / 2;
|
||||
diskann_id_t local_id = key_mapping_data_ptr[idx];
|
||||
|
||||
const diskann_key_t local_key = key_data_ptr[local_id];
|
||||
|
||||
if (local_key < key) {
|
||||
start = idx + 1;
|
||||
} else if (local_key > key) {
|
||||
end = idx;
|
||||
} else {
|
||||
return local_id;
|
||||
}
|
||||
}
|
||||
|
||||
return kInvalidId;
|
||||
}
|
||||
|
||||
diskann_key_t DiskAnnSearcherEntity::get_key(diskann_id_t id) const {
|
||||
const diskann_key_t *key_data_ptr =
|
||||
reinterpret_cast<const diskann_key_t *>(key_buffer_.data());
|
||||
|
||||
return key_data_ptr[id];
|
||||
}
|
||||
|
||||
const void *DiskAnnSearcherEntity::get_vector(diskann_id_t id) const {
|
||||
if (!vector_segment_) {
|
||||
LOG_ERROR("Vector segment is null");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint64_t sector_offset =
|
||||
DiskAnnUtil::get_node_sector(node_per_sector(), max_node_size(),
|
||||
DiskAnnUtil::kSectorSize, id) *
|
||||
DiskAnnUtil::kSectorSize;
|
||||
uint64_t within_sector_offset =
|
||||
(node_per_sector() == 0 ? 0 : (id % node_per_sector()) * max_node_size());
|
||||
uint64_t total_offset = sector_offset + within_sector_offset;
|
||||
|
||||
size_t read_size = meta_.element_size();
|
||||
const void *vec;
|
||||
if (ailego_unlikely(vector_segment_->read(total_offset, &vec, read_size) !=
|
||||
read_size)) {
|
||||
LOG_ERROR("Read vector from segment failed, id: %u, offset: %lu", id,
|
||||
total_offset);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return vec;
|
||||
}
|
||||
|
||||
std::pair<uint32_t, const diskann_id_t *> DiskAnnSearcherEntity::get_neighbors(
|
||||
diskann_id_t id) const {
|
||||
if (!vector_segment_) {
|
||||
return std::make_pair(0, nullptr);
|
||||
}
|
||||
|
||||
uint64_t read_sector_offset =
|
||||
DiskAnnUtil::get_node_sector(node_per_sector(), max_node_size(),
|
||||
DiskAnnUtil::kSectorSize, id) *
|
||||
DiskAnnUtil::kSectorSize;
|
||||
uint64_t node_vec_offset =
|
||||
read_sector_offset +
|
||||
(node_per_sector() == 0 ? 0 : (id % node_per_sector()) * max_node_size());
|
||||
|
||||
const void *data;
|
||||
if (ailego_unlikely(
|
||||
vector_segment_->read(node_vec_offset, &data, max_node_size()) !=
|
||||
max_node_size())) {
|
||||
LOG_ERROR("Read neighbors from segment failed");
|
||||
return {0, nullptr};
|
||||
}
|
||||
|
||||
const uint8_t *data_ptr = reinterpret_cast<const uint8_t *>(data);
|
||||
const diskann_id_t *node_neighbor =
|
||||
reinterpret_cast<const diskann_id_t *>(data_ptr + meta_.element_size());
|
||||
|
||||
auto neighbor_num = *node_neighbor;
|
||||
|
||||
return std::make_pair(neighbor_num, node_neighbor + 1);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
#include "diskann_entity.h"
|
||||
#include "diskann_file_reader.h"
|
||||
#include "diskann_pq_table.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnSearcherEntity : public DiskAnnEntity {
|
||||
public:
|
||||
using Pointer = std::shared_ptr<DiskAnnSearcherEntity>;
|
||||
using SegmentPointer = IndexStorage::Segment::Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnSearcherEntity() = default;
|
||||
virtual ~DiskAnnSearcherEntity() = default;
|
||||
|
||||
public:
|
||||
virtual const DiskAnnEntity::Pointer clone() const override;
|
||||
|
||||
int load(const IndexMeta &meta, IndexStorage::Pointer storage);
|
||||
int load_pq_segment();
|
||||
int load_header_segment();
|
||||
int load_vector_segment();
|
||||
int load_key_segment();
|
||||
int load_key_mapping_segment();
|
||||
int load_entrypoint_segment();
|
||||
|
||||
PQTable::Pointer get_pq_table() {
|
||||
return pq_table_;
|
||||
}
|
||||
|
||||
IndexStorage::Pointer get_storage() {
|
||||
return storage_;
|
||||
}
|
||||
|
||||
SegmentPointer get_vector_segment() {
|
||||
return vector_segment_;
|
||||
}
|
||||
|
||||
std::vector<diskann_id_t> &entrypoints() {
|
||||
return entrypoints_;
|
||||
}
|
||||
|
||||
std::pair<uint32_t, const diskann_id_t *> get_neighbors(
|
||||
diskann_id_t id) const override;
|
||||
|
||||
diskann_id_t get_id(diskann_key_t key) const override;
|
||||
diskann_key_t get_key(diskann_id_t id) const override;
|
||||
const void *get_vector(diskann_id_t id) const override;
|
||||
|
||||
private:
|
||||
DiskAnnSearcherEntity(
|
||||
const DiskAnnMetaHeader &meta_header, const DiskAnnPqMeta &pq_meta,
|
||||
const SegmentPointer &meta_segment, const SegmentPointer &pq_meta_segment,
|
||||
const SegmentPointer &pq_data_segment,
|
||||
const SegmentPointer &vector_segment, const SegmentPointer &key_segment,
|
||||
const SegmentPointer &key_mapping_segment,
|
||||
const SegmentPointer &entrypoint_segment, uint32_t num_threads,
|
||||
uint32_t list_size, uint32_t cache_nodes_num, bool warm_up,
|
||||
uint32_t beam_size, const IndexMeta meta, PQTable::Pointer pq_table,
|
||||
const std::string &key_buffer, const std::string &key_mapping_buffer,
|
||||
const std::vector<diskann_id_t> &entrypoints)
|
||||
: DiskAnnEntity(meta_header, pq_meta),
|
||||
meta_segment_(meta_segment),
|
||||
pq_meta_segment_(pq_meta_segment),
|
||||
pq_data_segment_(pq_data_segment),
|
||||
vector_segment_(vector_segment),
|
||||
key_segment_(key_segment),
|
||||
key_mapping_segment_(key_mapping_segment),
|
||||
entrypoint_segment_{entrypoint_segment},
|
||||
num_threads_{num_threads},
|
||||
list_size_{list_size},
|
||||
cache_nodes_num_{cache_nodes_num},
|
||||
warm_up_{warm_up},
|
||||
beam_size_{beam_size},
|
||||
meta_{meta},
|
||||
pq_table_{pq_table},
|
||||
key_buffer_{key_buffer},
|
||||
key_mapping_buffer_{key_mapping_buffer},
|
||||
entrypoints_{entrypoints} {}
|
||||
|
||||
IndexStorage::Pointer storage_{};
|
||||
|
||||
SegmentPointer meta_segment_{nullptr};
|
||||
SegmentPointer pq_meta_segment_{nullptr};
|
||||
SegmentPointer pq_data_segment_{nullptr};
|
||||
SegmentPointer vector_segment_{nullptr};
|
||||
SegmentPointer key_segment_{nullptr};
|
||||
SegmentPointer key_mapping_segment_{nullptr};
|
||||
SegmentPointer entrypoint_segment_{nullptr};
|
||||
|
||||
uint32_t num_threads_{1};
|
||||
uint32_t list_size_{200};
|
||||
uint32_t cache_nodes_num_{0};
|
||||
|
||||
bool warm_up_{false};
|
||||
uint32_t beam_size_{2};
|
||||
|
||||
IndexMeta meta_;
|
||||
|
||||
PQTable::Pointer pq_table_;
|
||||
std::string key_buffer_;
|
||||
std::string key_mapping_buffer_;
|
||||
std::vector<diskann_id_t> entrypoints_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,345 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "diskann_streamer.h"
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_index_provider.h"
|
||||
#include "diskann_indexer.h"
|
||||
#include "diskann_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
DiskAnnStreamer::DiskAnnStreamer() {}
|
||||
|
||||
DiskAnnStreamer::~DiskAnnStreamer() {}
|
||||
|
||||
int DiskAnnStreamer::init(const IndexMeta &meta,
|
||||
const ailego::Params &search_params) {
|
||||
meta_ = meta;
|
||||
search_params.get(PARAM_DISKANN_SEARCHER_LIST_SIZE, &list_size_);
|
||||
search_params.get(PARAM_DISKANN_SEARCHER_CACHE_NODE_NUM, &cache_nodes_num_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void DiskAnnStreamer::print_debug_info() {}
|
||||
|
||||
int DiskAnnStreamer::cleanup() {
|
||||
LOG_INFO("Begin DiskAnnStreamer:cleanup");
|
||||
|
||||
LOG_INFO("End DiskAnnStreamer:cleanup");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::open(IndexStorage::Pointer storage) {
|
||||
LOG_INFO("DiskAnnStreamer::load Begin");
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
int ret = IndexHelper::DeserializeFromStorage(storage.get(), &meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to deserialize meta from storage");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = entity_.load(meta_, storage);
|
||||
if (ret != 0) {
|
||||
LOG_INFO("Searcher Entity Load Failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
diskann_indexer_ = std::make_shared<DiskAnnIndexer>(meta_);
|
||||
|
||||
int res = diskann_indexer_->init(entity_);
|
||||
if (res != 0) {
|
||||
return res;
|
||||
}
|
||||
|
||||
if (cache_nodes_num_ != 0) {
|
||||
std::vector<diskann_id_t> node_list;
|
||||
LOG_INFO("Caching %u nodes around medoid(s)", cache_nodes_num_);
|
||||
|
||||
diskann_indexer_->cache_bfs_levels(cache_nodes_num_, node_list);
|
||||
|
||||
diskann_indexer_->load_cache_list(node_list);
|
||||
|
||||
node_list.clear();
|
||||
node_list.shrink_to_fit();
|
||||
}
|
||||
|
||||
measure_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!measure_) {
|
||||
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
ret = measure_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
if (measure_->query_metric()) {
|
||||
measure_ = measure_->query_metric();
|
||||
}
|
||||
|
||||
stats_.set_loaded_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
state_ = STATE_LOADED;
|
||||
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
LOG_INFO("DiskAnnStreamer::load Done");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::unload() {
|
||||
LOG_INFO("DiskAnnStreamer unload index");
|
||||
|
||||
state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::update_context(DiskAnnContext *ctx) const {
|
||||
const DiskAnnEntity::Pointer entity = entity_.clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone search context entity");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return ctx->update_context(DiskAnnContext::kSearcherContext, meta_, measure_,
|
||||
entity, magic_);
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
// do search
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
// Context is pooled per index type. When switching between DiskAnn indexes
|
||||
// with different element sizes (e.g., fp16 vs fp32), the cached context has
|
||||
// undersized buffers. Recreate it to ensure correct buffer allocations.
|
||||
if (ctx->magic() != magic_) {
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->knn_search(ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::search_bf_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer, recreate it
|
||||
//! to ensure buffers are correctly sized for this index's parameters.
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->linear_search(ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to DiskAnnContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(p_keys.size() != count)) {
|
||||
LOG_ERROR("The size of p_keys is not equal to count");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer, recreate it
|
||||
//! to ensure buffers are correctly sized for this index's parameters.
|
||||
uint32_t saved_topk = ctx->topk();
|
||||
context = create_context();
|
||||
if (!context) {
|
||||
LOG_ERROR("Failed to recreate context for current streamer");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx = dynamic_cast<DiskAnnContext *>(context.get());
|
||||
ctx->set_topk(saved_topk);
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
ctx->reset_query(query);
|
||||
|
||||
diskann_indexer_->keys_search(p_keys[i], ctx);
|
||||
|
||||
ctx->topk_to_result(i);
|
||||
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::get_vector(uint64_t key, Context::Pointer &context,
|
||||
std::string &vector) const {
|
||||
return diskann_indexer_->get_vector(key, context, vector);
|
||||
}
|
||||
|
||||
const void *DiskAnnStreamer::get_vector_by_id(uint32_t id) const {
|
||||
// DiskAnn vectors are stored on disk in sector format;
|
||||
// a const void* access requires sector I/O via create_context
|
||||
// Return nullptr to indicate this path is not supported.
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
int DiskAnnStreamer::get_vector_by_id(const uint32_t id,
|
||||
IndexStorage::MemoryBlock &block) const {
|
||||
// Lazily create a reusable context for fetch operations
|
||||
if (!fetch_ctx_) {
|
||||
fetch_ctx_ = create_context();
|
||||
if (!fetch_ctx_) {
|
||||
LOG_ERROR("Failed to create context for get_vector_by_id");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
int ret = diskann_indexer_->get_vector(id, fetch_ctx_, fetch_vector_buffer_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to get vector by id: %u", id);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
block.reset((void *)fetch_vector_buffer_.data());
|
||||
return 0;
|
||||
}
|
||||
|
||||
IndexSearcher::Provider::Pointer DiskAnnStreamer::create_provider(void) const {
|
||||
if (state_ != STATE_LOADED) {
|
||||
LOG_ERROR("Load the index first before creating a provider");
|
||||
return nullptr;
|
||||
}
|
||||
const DiskAnnEntity::Pointer entity = entity_.clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone DiskAnn entity for provider");
|
||||
return nullptr;
|
||||
}
|
||||
return IndexProvider::Pointer(new (std::nothrow) DiskAnnIndexProvider(
|
||||
meta_, entity, "DiskAnnStreamer"));
|
||||
}
|
||||
|
||||
IndexSearcher::Context::Pointer DiskAnnStreamer::create_context() const {
|
||||
const DiskAnnEntity::Pointer search_ctx_entity = entity_.clone();
|
||||
if (!search_ctx_entity) {
|
||||
LOG_ERROR("Failed to create search context entity");
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
DiskAnnContext *ctx =
|
||||
new (std::nothrow) DiskAnnContext(meta_, measure_, search_ctx_entity);
|
||||
if (ctx == nullptr) {
|
||||
LOG_ERROR("Failed to allocate DiskAnn Context");
|
||||
return Context::Pointer();
|
||||
}
|
||||
if (ailego_unlikely(ctx->init(
|
||||
DiskAnnContext::kSearcherContext, search_ctx_entity->max_degree(),
|
||||
search_ctx_entity->pq_chunk_num(), meta_.element_size())) != 0) {
|
||||
LOG_ERROR("Init DiskAnn Context failed");
|
||||
delete ctx;
|
||||
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
ctx->set_list_size(list_size_);
|
||||
|
||||
return Context::Pointer(ctx);
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_STREAMER(DiskAnnStreamer);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,180 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_context.h"
|
||||
#include "diskann_indexer.h"
|
||||
|
||||
class LinuxAlignedFileReader;
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnStreamer : public IndexStreamer {
|
||||
public:
|
||||
using ContextPointer = IndexStreamer::Context::Pointer;
|
||||
|
||||
public:
|
||||
DiskAnnStreamer(void);
|
||||
~DiskAnnStreamer(void);
|
||||
|
||||
DiskAnnStreamer(const DiskAnnStreamer &) = delete;
|
||||
DiskAnnStreamer &operator=(const DiskAnnStreamer &) = delete;
|
||||
|
||||
protected:
|
||||
//! Initialize Searcher
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Searcher
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Load Index from storage
|
||||
int open(IndexStorage::Pointer storage) override;
|
||||
|
||||
//! Unload index from storage
|
||||
int unload(void) override;
|
||||
|
||||
//! KNN Search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! KNN Search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, ContextPointer &context) const override;
|
||||
|
||||
//! Linear Search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear Search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, ContextPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query, const uint32_t sparse_count,
|
||||
const uint32_t *sparse_indices,
|
||||
const void *sparse_query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, &sparse_count, sparse_indices,
|
||||
sparse_query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(
|
||||
const void * /*query*/, const uint32_t * /*sparse_count*/,
|
||||
const uint32_t * /*sparse_indices*/, const void * /*sparse_query*/,
|
||||
const std::vector<std::vector<uint64_t>> & /*p_keys*/,
|
||||
const IndexQueryMeta & /*qmeta*/, uint32_t /*count*/,
|
||||
ContextPointer & /*context*/) const override {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Get vector by key
|
||||
int get_vector(uint64_t key, Context::Pointer &context,
|
||||
std::string &vector) const override;
|
||||
|
||||
//! Fetch vector by id
|
||||
const void *get_vector_by_id(uint32_t id) const override;
|
||||
|
||||
//! Fetch vector by id into memory block
|
||||
int get_vector_by_id(const uint32_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
//! Create a searcher context
|
||||
ContextPointer create_context() const override;
|
||||
|
||||
//! Create a vector iterator backed by the on-disk vector segment.
|
||||
//! Used by the merge code path (``MixedStreamerReducer``) to walk every
|
||||
//! vector held by this streamer.
|
||||
IndexSearcher::Provider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Retrieve statistics
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
virtual int flush(uint64_t /*check_point*/) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual int close(void) override {
|
||||
return this->unload();
|
||||
}
|
||||
|
||||
void print_debug_info() override;
|
||||
|
||||
private:
|
||||
template <typename T, typename LabelT = uint32_t>
|
||||
int search_disk_index(const std::string &query_file,
|
||||
const uint32_t num_nodes_to_cache,
|
||||
const uint32_t recall_at, const uint32_t beamwidth);
|
||||
|
||||
//! To share ctx across streamer/searcher, we need to update the context for
|
||||
//! current streamer/searcher
|
||||
int update_context(DiskAnnContext *ctx) const;
|
||||
|
||||
private:
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
|
||||
|
||||
IndexMetric::Pointer measure_{};
|
||||
IndexMeta meta_{};
|
||||
ailego::Params params_{};
|
||||
|
||||
uint32_t list_size_{200};
|
||||
uint32_t cache_nodes_num_{0};
|
||||
|
||||
bool warm_up_{false};
|
||||
uint32_t beam_size_{2};
|
||||
|
||||
DiskAnnIndexer::Pointer diskann_indexer_{nullptr};
|
||||
DiskAnnSearcherEntity entity_{};
|
||||
|
||||
// Mutable members for get_vector_by_id (caches context and buffer)
|
||||
mutable ContextPointer fetch_ctx_{};
|
||||
mutable std::string fetch_vector_buffer_;
|
||||
|
||||
uint32_t magic_{0U};
|
||||
|
||||
Stats stats_;
|
||||
State state_{STATE_INIT};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,221 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "diskann_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class DiskAnnUtil {
|
||||
public:
|
||||
static constexpr uint64_t kSectorSize = 4096;
|
||||
static constexpr uint64_t kMaxSectorReadNum = 128;
|
||||
|
||||
public:
|
||||
static inline size_t div_round_up(size_t x, size_t y) {
|
||||
return (x / y + (x % y != 0));
|
||||
}
|
||||
|
||||
static inline size_t round_up(size_t x, size_t y) {
|
||||
return div_round_up(x, y) * y;
|
||||
}
|
||||
|
||||
static inline void alloc_aligned(void **ptr, size_t size, size_t align) {
|
||||
*ptr = ::aligned_alloc(align, size);
|
||||
}
|
||||
|
||||
static inline void free_aligned(void *ptr) {
|
||||
if (ptr == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
free(ptr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline void convert_vector_to_residual(T *data, uint32_t blocksize_,
|
||||
size_t dim, void *centroid) {
|
||||
const T *centroid_ptr = reinterpret_cast<const T *>(centroid);
|
||||
for (size_t i = 0; i < blocksize_; i++) {
|
||||
for (uint64_t d = 0; d < dim; d++) {
|
||||
float data_float = data[i * dim + d];
|
||||
data_float -= centroid_ptr[d];
|
||||
data[i * dim + d] = data_float;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void convert_types_uint32_to_uint8(const uint32_t *src,
|
||||
uint8_t *dest, size_t npts,
|
||||
size_t dim) {
|
||||
for (size_t i = 0; i < npts; i++) {
|
||||
for (size_t j = 0; j < dim; j++) {
|
||||
dest[i * dim + j] = src[i * dim + j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline uint64_t get_node_sector(uint32_t node_per_sector,
|
||||
uint32_t max_nodesize_,
|
||||
uint32_t sectorsize_,
|
||||
diskann_id_t node_id) {
|
||||
return (node_per_sector > 0
|
||||
? node_id / node_per_sector
|
||||
: node_id * div_round_up(max_nodesize_, sectorsize_));
|
||||
}
|
||||
|
||||
static inline uint32_t *offset_to_node_neighbor(uint8_t *node_buf,
|
||||
uint32_t elementsize_) {
|
||||
return (uint32_t *)(node_buf + elementsize_);
|
||||
}
|
||||
|
||||
static inline uint8_t *offset_to_node(uint32_t node_per_sector,
|
||||
uint32_t max_nodesize_,
|
||||
uint8_t *sector_buf,
|
||||
diskann_id_t node_id) {
|
||||
return sector_buf + (node_per_sector == 0
|
||||
? 0
|
||||
: (node_id % node_per_sector) * max_nodesize_);
|
||||
}
|
||||
|
||||
static inline const uint8_t *offset_to_node_const(uint32_t node_per_sector,
|
||||
uint32_t max_nodesize_,
|
||||
const uint8_t *sector_buf,
|
||||
diskann_id_t node_id) {
|
||||
return sector_buf + (node_per_sector == 0
|
||||
? 0
|
||||
: (node_id % node_per_sector) * max_nodesize_);
|
||||
}
|
||||
};
|
||||
|
||||
//! Neighbor
|
||||
struct Neighbor {
|
||||
public:
|
||||
Neighbor() = default;
|
||||
|
||||
Neighbor(diskann_id_t id, float distance)
|
||||
: id{id}, distance{distance}, expanded(false) {}
|
||||
|
||||
inline bool operator<(const Neighbor &other) const {
|
||||
return distance < other.distance ||
|
||||
(distance == other.distance && id < other.id);
|
||||
}
|
||||
|
||||
inline bool operator==(const Neighbor &other) const {
|
||||
return (id == other.id);
|
||||
}
|
||||
|
||||
public:
|
||||
diskann_id_t id;
|
||||
float distance;
|
||||
bool expanded;
|
||||
};
|
||||
|
||||
//! NeighborPriorityQueue
|
||||
class NeighborPriorityQueue {
|
||||
public:
|
||||
NeighborPriorityQueue() : size_(0), capacity_(0), cur_(0) {}
|
||||
|
||||
explicit NeighborPriorityQueue(size_t capacity)
|
||||
: size_(0), capacity_(capacity), cur_(0), data_(capacity + 1) {}
|
||||
|
||||
void insert(const Neighbor &nbr) {
|
||||
if (size_ == capacity_ && data_[size_ - 1] < nbr) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t low = 0, high = size_;
|
||||
while (low < high) {
|
||||
size_t mid = (low + high) >> 1;
|
||||
if (nbr < data_[mid]) {
|
||||
high = mid;
|
||||
} else if (data_[mid].id == nbr.id) {
|
||||
return;
|
||||
} else {
|
||||
low = mid + 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (low < capacity_) {
|
||||
std::memmove(&data_[low + 1], &data_[low],
|
||||
(size_ - low) * sizeof(Neighbor));
|
||||
}
|
||||
|
||||
data_[low] = {nbr.id, nbr.distance};
|
||||
if (size_ < capacity_) {
|
||||
size_++;
|
||||
}
|
||||
|
||||
if (low < cur_) {
|
||||
cur_ = low;
|
||||
}
|
||||
}
|
||||
|
||||
Neighbor closest_unexpanded() {
|
||||
data_[cur_].expanded = true;
|
||||
size_t pre = cur_;
|
||||
while (cur_ < size_ && data_[cur_].expanded) {
|
||||
cur_++;
|
||||
}
|
||||
return data_[pre];
|
||||
}
|
||||
|
||||
bool has_unexpanded_node() const {
|
||||
return cur_ < size_;
|
||||
}
|
||||
|
||||
size_t size() const {
|
||||
return size_;
|
||||
}
|
||||
|
||||
size_t capacity() const {
|
||||
return capacity_;
|
||||
}
|
||||
|
||||
void reserve(size_t capacity) {
|
||||
if (capacity + 1 > data_.size()) {
|
||||
data_.resize(capacity + 1);
|
||||
}
|
||||
capacity_ = capacity;
|
||||
}
|
||||
|
||||
Neighbor &operator[](size_t i) {
|
||||
return data_[i];
|
||||
}
|
||||
|
||||
Neighbor operator[](size_t i) const {
|
||||
return data_[i];
|
||||
}
|
||||
|
||||
void sort() {
|
||||
std::sort(data_.begin(), data_.begin() + size_);
|
||||
}
|
||||
|
||||
void clear() {
|
||||
size_ = 0;
|
||||
cur_ = 0;
|
||||
}
|
||||
|
||||
private:
|
||||
size_t size_;
|
||||
size_t capacity_;
|
||||
size_t cur_;
|
||||
std::vector<Neighbor> data_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,320 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <ailego/io/mmap_file.h>
|
||||
#include <framework/index_framework.h>
|
||||
#include <zvec/core/framework/index_holder.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
enum VecsBitMapIndex {
|
||||
BITMAP_INDEX_KEY = 0,
|
||||
BITMAP_INDEX_DENSE = 1,
|
||||
BITMAP_INDEX_SPARSE = 2,
|
||||
BITMAP_INDEX_PARTITION = 3,
|
||||
BITMAP_INDEX_TAGLIST = 4
|
||||
};
|
||||
|
||||
#pragma pack(4)
|
||||
struct VecsHeaderDetect {
|
||||
uint64_t num_vecs;
|
||||
uint16_t reserved; // in v0, it's meta size
|
||||
uint16_t version; // 0 for v0 format, 1 for new format
|
||||
uint8_t meta_buf[0];
|
||||
};
|
||||
#pragma pack()
|
||||
|
||||
#pragma pack(4)
|
||||
struct VecsHeader {
|
||||
uint64_t num_vecs;
|
||||
uint32_t meta_size;
|
||||
uint8_t meta_buf[0];
|
||||
};
|
||||
#pragma pack()
|
||||
|
||||
#pragma pack(4)
|
||||
struct VecsHeaderV1 {
|
||||
uint64_t num_vecs;
|
||||
uint16_t meta_size_v1;
|
||||
uint16_t version;
|
||||
uint32_t meta_size;
|
||||
uint64_t bitmap; // set for data section
|
||||
uint64_t key_offset; // offset for key
|
||||
uint64_t key_size; // size for key
|
||||
uint64_t dense_offset; // offset for dense
|
||||
uint64_t dense_size; // size for dense
|
||||
uint64_t sparse_offset; // offset for sparse
|
||||
uint64_t sparse_size; // size for sparse
|
||||
uint64_t partition_offset; // offset for partition
|
||||
uint64_t partition_size; // size for partition
|
||||
uint64_t taglist_offset; // offset for taglist
|
||||
uint64_t taglist_size; // size for taglist
|
||||
uint8_t meta_buf[0];
|
||||
};
|
||||
#pragma pack()
|
||||
|
||||
class DiskAnnVecsReader {
|
||||
public:
|
||||
DiskAnnVecsReader()
|
||||
: mmap_file_(),
|
||||
index_meta_(),
|
||||
num_vecs_(0),
|
||||
vector_base_(nullptr),
|
||||
key_base_(nullptr),
|
||||
sparse_base_meta_{nullptr},
|
||||
sparse_base_data_{nullptr},
|
||||
partition_base_{nullptr},
|
||||
taglist_base_meta_{nullptr},
|
||||
taglist_base_data_{nullptr},
|
||||
taglist_size_{0} {}
|
||||
|
||||
void set_measure(const std::string &name, const IndexParams ¶ms) {
|
||||
index_meta_.set_measure(name, 0, params);
|
||||
}
|
||||
|
||||
bool load(const std::string &fname) {
|
||||
return load(fname.c_str());
|
||||
}
|
||||
|
||||
bool load(const char *fname) {
|
||||
if (!fname) {
|
||||
std::cerr << "Load fname is nullptr" << std::endl;
|
||||
return false;
|
||||
}
|
||||
if (!mmap_file_.open(fname, true)) {
|
||||
std::cerr << "Open file error: " << fname << std::endl;
|
||||
return false;
|
||||
}
|
||||
if (mmap_file_.size() < sizeof(VecsHeaderDetect)) {
|
||||
std::cerr << "File size is too small: " << mmap_file_.size() << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
const VecsHeaderDetect *header =
|
||||
reinterpret_cast<const VecsHeaderDetect *>(mmap_file_.region());
|
||||
|
||||
if (header->version == 0) {
|
||||
return load_v0();
|
||||
} else if (header->version == 1) {
|
||||
return load_v1();
|
||||
}
|
||||
|
||||
std::cerr << "Can not recognize version: " << header->version << std::endl;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool load_v0() {
|
||||
const VecsHeader *header =
|
||||
reinterpret_cast<const VecsHeader *>(mmap_file_.region());
|
||||
// check
|
||||
num_vecs_ = header->num_vecs;
|
||||
|
||||
// deserialize
|
||||
bool bret = index_meta_.deserialize(&header->meta_buf, header->meta_size);
|
||||
if (!bret) {
|
||||
std::cerr << "deserialize index meta error." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!index_meta_.hybrid_vector()) {
|
||||
if ((mmap_file_.size() - sizeof(*header) - header->meta_size) %
|
||||
num_vecs_ !=
|
||||
0) {
|
||||
std::cerr << "input file foramt check error." << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!index_meta_.hybrid_vector()) {
|
||||
vector_base_ =
|
||||
reinterpret_cast<const char *>(header + 1) + header->meta_size;
|
||||
key_base_ = reinterpret_cast<const uint64_t *>(
|
||||
vector_base_ + num_vecs_ * index_meta_.element_size());
|
||||
} else {
|
||||
vector_base_ =
|
||||
reinterpret_cast<const char *>(header + 1) + header->meta_size;
|
||||
key_base_ = reinterpret_cast<const uint64_t *>(
|
||||
vector_base_ + num_vecs_ * index_meta_.element_size());
|
||||
sparse_base_meta_ = reinterpret_cast<const char *>(key_base_ + num_vecs_);
|
||||
sparse_base_data_ = reinterpret_cast<const char *>(
|
||||
sparse_base_meta_ + num_vecs_ * sizeof(uint64_t));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool load_v1() {
|
||||
const VecsHeaderV1 *header =
|
||||
reinterpret_cast<const VecsHeaderV1 *>(mmap_file_.region());
|
||||
// check
|
||||
num_vecs_ = header->num_vecs;
|
||||
|
||||
// deserialize
|
||||
bool bret = index_meta_.deserialize(&header->meta_buf, header->meta_size);
|
||||
if (!bret) {
|
||||
std::cerr << "deserialize index meta error." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
const char *data_base_ptr =
|
||||
reinterpret_cast<const char *>(header + 1) + header->meta_size;
|
||||
|
||||
vector_base_ = reinterpret_cast<const char *>(data_base_ptr);
|
||||
key_base_ = reinterpret_cast<const uint64_t *>(
|
||||
vector_base_ + num_vecs_ * index_meta_.element_size());
|
||||
|
||||
if (header->sparse_offset != -1LLU) {
|
||||
sparse_base_meta_ = data_base_ptr + header->sparse_offset;
|
||||
sparse_base_data_ = sparse_base_meta_ + num_vecs_ * sizeof(uint64_t);
|
||||
}
|
||||
|
||||
if (header->partition_offset != -1LLU) {
|
||||
partition_base_ = reinterpret_cast<const uint32_t *>(
|
||||
data_base_ptr + header->partition_offset);
|
||||
}
|
||||
|
||||
if (header->taglist_offset != -1LLU) {
|
||||
taglist_base_meta_ = data_base_ptr + header->taglist_offset;
|
||||
taglist_base_data_ = taglist_base_meta_ + num_vecs_;
|
||||
taglist_size_ = header->taglist_size;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t num_vecs() const {
|
||||
return num_vecs_;
|
||||
}
|
||||
|
||||
const void *vector_base() const {
|
||||
return vector_base_;
|
||||
}
|
||||
|
||||
const uint64_t *key_base() const {
|
||||
return key_base_;
|
||||
}
|
||||
|
||||
const IndexMeta &index_meta() const {
|
||||
return index_meta_;
|
||||
}
|
||||
|
||||
uint64_t get_key(size_t index) const {
|
||||
return key_base_[index];
|
||||
}
|
||||
|
||||
const void *get_vector(size_t index) const {
|
||||
return vector_base_ + index * index_meta_.element_size();
|
||||
}
|
||||
|
||||
uint32_t get_sparse_count(size_t index) const {
|
||||
if (index_meta_.hybrid_vector()) {
|
||||
auto sparse_data_meta = sparse_base_meta_ + index * sizeof(uint64_t);
|
||||
uint64_t sparse_offset = *((uint64_t *)sparse_data_meta);
|
||||
uint32_t sparse_count =
|
||||
*((uint32_t *)(sparse_base_data_ + sparse_offset));
|
||||
|
||||
return sparse_count;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const uint32_t *get_sparse_indices(size_t index) const {
|
||||
if (index_meta_.hybrid_vector()) {
|
||||
auto sparse_data_meta = sparse_base_meta_ + index * sizeof(uint64_t);
|
||||
uint64_t sparse_offset = *((uint64_t *)sparse_data_meta);
|
||||
uint32_t *sparse_indices =
|
||||
(uint32_t *)(sparse_base_data_ + sparse_offset + sizeof(uint32_t));
|
||||
|
||||
return sparse_indices;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const void *get_sparse_data(size_t index) const {
|
||||
if (index_meta_.hybrid_vector()) {
|
||||
auto sparse_data_meta = sparse_base_meta_ + index * sizeof(uint64_t);
|
||||
uint64_t sparse_offset = *((uint64_t *)sparse_data_meta);
|
||||
uint32_t sparse_count =
|
||||
*((uint32_t *)(sparse_base_data_ + sparse_offset));
|
||||
void *sparse_data =
|
||||
(uint32_t *)(sparse_base_data_ + sparse_offset + sizeof(uint32_t) +
|
||||
sparse_count * sizeof(uint32_t));
|
||||
|
||||
return sparse_data;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
size_t get_total_sparse_count(void) const {
|
||||
size_t total_sparse_count = 0;
|
||||
for (size_t i = 0; i < num_vecs_; ++i) {
|
||||
total_sparse_count += get_sparse_count(i);
|
||||
}
|
||||
|
||||
return total_sparse_count;
|
||||
}
|
||||
|
||||
bool has_taglist(void) const {
|
||||
return taglist_base_meta_ != nullptr;
|
||||
}
|
||||
|
||||
uint64_t get_taglist_count(size_t index) const {
|
||||
if (!taglist_base_data_ || !taglist_base_meta_) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint64_t taglist_count = *reinterpret_cast<const uint64_t *>(
|
||||
taglist_base_data_ + taglist_base_meta_[index]);
|
||||
return taglist_count;
|
||||
}
|
||||
|
||||
const uint64_t *get_taglist(size_t index) const {
|
||||
if (!taglist_base_data_ || !taglist_base_meta_) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return reinterpret_cast<const uint64_t *>(taglist_base_data_ +
|
||||
taglist_base_meta_[index]) +
|
||||
1;
|
||||
}
|
||||
|
||||
const void *get_taglist_data(size_t &size) const {
|
||||
size = taglist_size_;
|
||||
|
||||
return taglist_base_meta_;
|
||||
}
|
||||
|
||||
private:
|
||||
ailego::MMapFile mmap_file_;
|
||||
IndexMeta index_meta_;
|
||||
size_t num_vecs_;
|
||||
const char *vector_base_;
|
||||
const uint64_t *key_base_;
|
||||
const char *sparse_base_meta_;
|
||||
const char *sparse_base_data_;
|
||||
const uint32_t *partition_base_;
|
||||
const char *taglist_base_meta_;
|
||||
const char *taglist_base_data_;
|
||||
uint64_t taglist_size_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,419 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <random>
|
||||
#include <tuple>
|
||||
#include <ailego/container/bloom_filter.h>
|
||||
#include <ailego/utility/bitset_helper.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/framework/index_logger.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
struct VisitFilterHeader {
|
||||
VisitFilterHeader() : maxDocCnt(0), maxScanNum(0) {}
|
||||
uint64_t maxDocCnt;
|
||||
uint64_t maxScanNum;
|
||||
};
|
||||
|
||||
constexpr int PROXIMA_DISKANN_VISITFILTER_CUSTOM_PARAMS_INDEX_NEGPROB = 0;
|
||||
|
||||
class VisitBloomFilter {
|
||||
public:
|
||||
static constexpr int mode = 1;
|
||||
|
||||
static constexpr int N = 5;
|
||||
struct Context {
|
||||
Context()
|
||||
: mt(std::chrono::system_clock::now().time_since_epoch().count()) {};
|
||||
VisitFilterHeader h;
|
||||
std::mt19937 mt;
|
||||
ailego::BloomFilter<N> *filter{nullptr};
|
||||
int offset[N] = {0};
|
||||
};
|
||||
#define BLOOM_FILTER_HASH_BITS_OFFSETS(i) \
|
||||
i + c->offset[0], i + c->offset[1], i + c->offset[2], i + c->offset[3], \
|
||||
i + c->offset[4]
|
||||
|
||||
VisitBloomFilter() = delete;
|
||||
|
||||
inline static void set_visited(Context *c, id_t idx) {
|
||||
c->filter->force_insert(BLOOM_FILTER_HASH_BITS_OFFSETS(idx));
|
||||
return;
|
||||
}
|
||||
|
||||
inline static void *get_visited(Context *, id_t) {
|
||||
// TODO
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
inline static bool visited(Context *c, id_t idx) {
|
||||
return c->filter->has(BLOOM_FILTER_HASH_BITS_OFFSETS(idx));
|
||||
}
|
||||
|
||||
inline static int set_max_scan_num(Context *c, uint64_t maxScanNum) {
|
||||
if (maxScanNum == c->h.maxScanNum) {
|
||||
return 0;
|
||||
}
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
if (c->filter->reset(maxScanNum, c->filter->probability()) != 0) {
|
||||
LOG_ERROR("reset BloomFilter failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
genRandomHashBits(c);
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void clear(Context *c) {
|
||||
c->filter->clear();
|
||||
return;
|
||||
}
|
||||
|
||||
inline static bool reset(Context *c, uint64_t maxDocCnt,
|
||||
uint64_t max_scan_num) {
|
||||
if (ailego_unlikely(maxDocCnt > c->h.maxDocCnt ||
|
||||
max_scan_num > c->h.maxScanNum)) {
|
||||
// Create a new one, if failed, we can reuse the old one
|
||||
auto filter = new (std::nothrow) ailego::BloomFilter<VisitBloomFilter::N>(
|
||||
max_scan_num, c->filter->probability());
|
||||
if (ailego_unlikely(filter == nullptr)) {
|
||||
LOG_ERROR("reset bloomfilter failed, maxScanNum %zu prob %f",
|
||||
(size_t)max_scan_num, c->filter->probability());
|
||||
c->filter->clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
delete c->filter;
|
||||
c->filter = filter;
|
||||
c->h.maxScanNum = max_scan_num;
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
genRandomHashBits(c);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
inline static void genRandomHashBits(Context *c) {
|
||||
std::uniform_int_distribution<int> dt(0, c->h.maxDocCnt);
|
||||
for (size_t i = 0; i < sizeof(c->offset) / sizeof(c->offset[0]); ++i) {
|
||||
int r = dt(c->mt);
|
||||
size_t j = 0;
|
||||
do { // gen distinct number
|
||||
for (j = 0; j < i; ++j) {
|
||||
if (c->offset[j] == r) {
|
||||
r = dt(c->mt);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} while (j < i);
|
||||
c->offset[i] = r;
|
||||
}
|
||||
std::sort(c->offset, c->offset + N);
|
||||
}
|
||||
|
||||
template <class... T>
|
||||
static int init(Context *, void **ctx, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum, std::tuple<T...> &&tpl) {
|
||||
Context *c = new (std::nothrow) Context;
|
||||
if (c == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitBitMap failed");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
float p =
|
||||
std::get<PROXIMA_DISKANN_VISITFILTER_CUSTOM_PARAMS_INDEX_NEGPROB>(tpl);
|
||||
c->filter = new (std::nothrow)
|
||||
ailego::BloomFilter<VisitBloomFilter::N>(maxScanNum, p);
|
||||
if (c->filter == nullptr) {
|
||||
LOG_ERROR("New BloomFilter failed, reuse old one");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
genRandomHashBits(c);
|
||||
*ctx = c;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void destroy(Context *c) {
|
||||
delete c->filter;
|
||||
delete c;
|
||||
}
|
||||
#undef BLOOM_FILTER_HASH_BITS_OFFSETS
|
||||
}; // end of VisitBloomFilter
|
||||
|
||||
class VisitBitMap {
|
||||
public:
|
||||
static constexpr int mode = 2;
|
||||
|
||||
struct Context {
|
||||
VisitFilterHeader h;
|
||||
ailego::BitsetHelper bitset;
|
||||
char *buf{nullptr};
|
||||
};
|
||||
|
||||
VisitBitMap() = delete;
|
||||
|
||||
inline static void set_visited(Context *c, id_t idx) {
|
||||
c->bitset.set(idx);
|
||||
return;
|
||||
}
|
||||
|
||||
inline static void *get_visited(Context *c, id_t idx) {
|
||||
return &c->buf[idx >> 3];
|
||||
}
|
||||
|
||||
inline static bool visited(Context *c, id_t idx) {
|
||||
return c->bitset.test(idx);
|
||||
}
|
||||
|
||||
inline static int set_max_scan_num(Context *c, uint64_t maxScanNum) {
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void clear(Context *c) {
|
||||
c->bitset.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
inline static bool reset(Context *c, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum) {
|
||||
if (ailego_unlikely(maxDocCnt > c->h.maxDocCnt ||
|
||||
maxScanNum > c->h.maxScanNum)) {
|
||||
uint64_t len = ((maxDocCnt + 31) >> 5) << 2; // round to uint32_t
|
||||
auto buf = new (std::nothrow) char[len];
|
||||
if (buf == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitBitMap failed");
|
||||
c->bitset.clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
delete[] c->buf;
|
||||
c->buf = buf;
|
||||
memset(c->buf, 0, len);
|
||||
c->bitset.mount(c->buf, len);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class... T>
|
||||
static int init(Context *, void **ctx, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum, std::tuple<T...> &&tpl) {
|
||||
(void)tpl; // unsed warning
|
||||
Context *c = new (std::nothrow) Context;
|
||||
if (c == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitBitMap failed");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
uint64_t len = ((maxDocCnt + 31) >> 5) << 2; // round to uint32_t
|
||||
c->buf = new (std::nothrow) char[len];
|
||||
if (c->buf == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitBitMap failed, reuse old one");
|
||||
delete c;
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
memset(c->buf, 0, len);
|
||||
c->bitset.mount(c->buf, len);
|
||||
*ctx = c;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void destroy(Context *c) {
|
||||
delete[] c->buf;
|
||||
delete c;
|
||||
}
|
||||
}; // end of VisitBitMap
|
||||
|
||||
class VisitByteMap {
|
||||
public:
|
||||
static constexpr int mode = 3;
|
||||
struct Context {
|
||||
VisitFilterHeader h;
|
||||
uint8_t curNum{0};
|
||||
uint8_t *arr{nullptr};
|
||||
};
|
||||
|
||||
VisitByteMap() = delete;
|
||||
|
||||
inline static void set_visited(Context *c, id_t idx) {
|
||||
c->arr[idx] = c->curNum;
|
||||
return;
|
||||
}
|
||||
|
||||
inline static void *get_visited(Context *c, id_t idx) {
|
||||
return c->arr + idx;
|
||||
}
|
||||
|
||||
inline static bool visited(Context *c, id_t idx) {
|
||||
return c->arr[idx] == c->curNum;
|
||||
}
|
||||
|
||||
inline static int set_max_scan_num(Context *c, uint64_t maxScanNum) {
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void clear(Context *c) {
|
||||
c->curNum++;
|
||||
if (c->curNum == 0) {
|
||||
memset(c->arr, 0, c->h.maxDocCnt * sizeof(uint8_t));
|
||||
c->curNum = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
inline static bool reset(Context *c, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum) {
|
||||
if (ailego_unlikely(maxDocCnt > c->h.maxDocCnt ||
|
||||
maxScanNum > c->h.maxScanNum)) {
|
||||
auto arr = new (std::nothrow) uint8_t[maxDocCnt];
|
||||
if (arr != nullptr) {
|
||||
memset(arr, 0, maxDocCnt * sizeof(uint8_t));
|
||||
c->curNum = 1;
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
delete[] c->arr;
|
||||
c->arr = arr;
|
||||
return true;
|
||||
}
|
||||
LOG_ERROR("New memory in initVisitByteMap failed, reuse old one");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class... T>
|
||||
static int init(Context *, void **ctx, uint64_t maxDocCnt,
|
||||
uint64_t maxScanNum, std::tuple<T...> &&tpl) {
|
||||
(void)tpl; // unsed warning
|
||||
Context *c = new (std::nothrow) Context;
|
||||
if (c == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitByteMap failed");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
c->h.maxDocCnt = maxDocCnt;
|
||||
c->h.maxScanNum = maxScanNum;
|
||||
c->arr = new (std::nothrow) uint8_t[maxDocCnt];
|
||||
if (c->arr == nullptr) {
|
||||
LOG_ERROR("New memory in initVisitByteMap failed");
|
||||
delete c;
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
memset(c->arr, 0, maxDocCnt * sizeof(uint8_t));
|
||||
c->curNum = 1;
|
||||
*ctx = c;
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline static void destroy(Context *c) {
|
||||
delete[] c->arr;
|
||||
delete c;
|
||||
}
|
||||
}; // end of VisitByteMap
|
||||
|
||||
|
||||
#define PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(cls, impl, ctx, ...) \
|
||||
case cls::mode: \
|
||||
return cls::impl(static_cast<cls::Context *>(ctx), ##__VA_ARGS__);
|
||||
|
||||
#define PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(impl, ...) \
|
||||
switch (mode_) { \
|
||||
PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(VisitBloomFilter, impl, ctx_, \
|
||||
##__VA_ARGS__) \
|
||||
PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(VisitBitMap, impl, ctx_, \
|
||||
##__VA_ARGS__) \
|
||||
PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(VisitByteMap, impl, ctx_, \
|
||||
##__VA_ARGS__) \
|
||||
}
|
||||
|
||||
|
||||
// visit list will be called with high frequency,
|
||||
// so using switch instead of std::function or virtual class
|
||||
// funtion point, lambda, virtual class all cannot be inlined
|
||||
class VisitFilter {
|
||||
public:
|
||||
enum Mode {
|
||||
Default = 0,
|
||||
BloomFilter = VisitBloomFilter::mode,
|
||||
BitMap = VisitBitMap::mode,
|
||||
ByteMap = VisitByteMap::mode
|
||||
};
|
||||
|
||||
VisitFilter() : mode_(0), ctx_(nullptr) {};
|
||||
|
||||
inline bool visited(id_t idx) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(visited, idx);
|
||||
return true; // place holder
|
||||
}
|
||||
|
||||
inline void set_visited(id_t idx) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(set_visited, idx);
|
||||
}
|
||||
|
||||
inline void *get_visited(id_t idx) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(get_visited, idx);
|
||||
return nullptr; // place holder
|
||||
}
|
||||
|
||||
inline int set_max_scan_num(id_t idx) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(set_max_scan_num, idx);
|
||||
return 0; // place holder
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(clear);
|
||||
}
|
||||
|
||||
inline bool reset(uint64_t maxDocCnt, uint64_t maxScanNum) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(reset, maxDocCnt, maxScanNum);
|
||||
return true;
|
||||
}
|
||||
|
||||
inline void destroy() {
|
||||
if (ctx_ != nullptr) {
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(destroy);
|
||||
}
|
||||
}
|
||||
|
||||
int init(int mode, uint64_t maxDocCnt, uint64_t maxScanNum,
|
||||
float negativeProbility) {
|
||||
mode_ = mode;
|
||||
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(init, &ctx_, maxDocCnt, maxScanNum,
|
||||
std::make_tuple(negativeProbility));
|
||||
return 0; // place holder
|
||||
}
|
||||
|
||||
int get_mode(void) const {
|
||||
return mode_;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
VisitFilter(const VisitFilter &) = delete;
|
||||
VisitFilter &operator=(const VisitFilter &) = delete;
|
||||
|
||||
int mode_{0U}; // custom data for each method
|
||||
void *ctx_{nullptr};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -533,4 +533,4 @@ class HnswContext : public IndexContext {
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
} // namespace zvec
|
||||
Reference in New Issue
Block a user