feat: add diskann index (#369)

This commit is contained in:
rayx
2026-06-04 20:52:43 +08:00
committed by GitHub
parent c46efe1241
commit e720c1fd20
104 changed files with 13913 additions and 232 deletions
+36
View File
@@ -8,6 +8,42 @@ cc_directory(ivf)
cc_directory(hnsw)
cc_directory(hnsw_sparse)
cc_directory(vamana)
if(DISKANN_SUPPORTED)
message(STATUS "build diskann")
cc_directory(diskann)
else()
message(STATUS "not build diskann")
# Empty stub library for unsupported platforms
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/diskann_stub.cc
"// Stub implementation for unsupported platforms\n"
"// DiskAnn only supports Linux x86_64\n"
"namespace zvec { namespace core { /* empty namespace for compatibility */ } }\n"
)
if(MSVC)
# MSVC: STATIC-only stub to avoid creating an empty DLL with no exports
# (MSVC linker fails to produce an import library when there are zero exports)
cc_library(
NAME core_knn_diskann
STATIC STRICT ALWAYS_LINK
SRCS ${CMAKE_CURRENT_BINARY_DIR}/diskann_stub.cc
LIBS core_framework
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
VERSION "${PROXIMA_ZVEC_VERSION}"
)
else()
cc_library(
NAME core_knn_diskann
STATIC SHARED STRICT ALWAYS_LINK
SRCS ${CMAKE_CURRENT_BINARY_DIR}/diskann_stub.cc
LIBS core_framework
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
VERSION "${PROXIMA_ZVEC_VERSION}"
)
endif()
endif()
if(RABITQ_SUPPORTED)
message(STATUS "BUILD RABITQ")
cc_directory(hnsw_rabitq)
+96 -90
View File
@@ -19,189 +19,195 @@ namespace zvec {
namespace core {
//! General
static const std::string GENERAL_CLUSTER_COUNT =
"proxima.general.cluster.count";
static const std::string GENERAL_CLUSTER_COUNT = "zvec.general.cluster.count";
static const std::string GENERAL_THREAD_COUNT =
"proxima.general.cluster.thread_count";
"zvec.general.cluster.thread_count";
//! Optimize K-means
static const std::string OPTKMEANS_CLUSTER_COUNT =
"proxima.optkmeans.cluster.count";
"zvec.optkmeans.cluster.count";
static const std::string OPTKMEANS_CLUSTER_MAX_ITERATIONS =
"proxima.optkmeans.cluster.max_iterations";
"zvec.optkmeans.cluster.max_iterations";
static const std::string OPTKMEANS_CLUSTER_EPSILON =
"proxima.optkmeans.cluster.epsilon";
"zvec.optkmeans.cluster.epsilon";
static const std::string OPTKMEANS_CLUSTER_SHARD_FACTOR =
"proxima.optkmeans.cluster.shard_factor";
"zvec.optkmeans.cluster.shard_factor";
static const std::string OPTKMEANS_CLUSTER_PURGE_EMPTY =
"proxima.optkmeans.cluster.purge_empty";
"zvec.optkmeans.cluster.purge_empty";
static const std::string OPTKMEANS_CLUSTER_MARKOV_CHAIN_LENGTH =
"proxima.optkmeans.cluster.markov_chain_length";
"zvec.optkmeans.cluster.markov_chain_length";
static const std::string OPTKMEANS_CLUSTER_ASSUMPTION_FREE =
"proxima.optkmeans.cluster.assumption_free";
"zvec.optkmeans.cluster.assumption_free";
//! K-means
static const std::string KMEANS_CLUSTER_COUNT = "proxima.kmeans.cluster.count";
static const std::string KMEANS_CLUSTER_COUNT = "zvec.kmeans.cluster.count";
static const std::string KMEANS_CLUSTER_SHARD_FACTOR =
"proxima.kmeans.cluster.shard_factor";
static const std::string KMEANS_CLUSTER_EPSILON =
"proxima.kmeans.cluster.epsilon";
"zvec.kmeans.cluster.shard_factor";
static const std::string KMEANS_CLUSTER_EPSILON = "zvec.kmeans.cluster.epsilon";
static const std::string KMEANS_CLUSTER_MAX_ITERATIONS =
"proxima.kmeans.cluster.max_iterations";
"zvec.kmeans.cluster.max_iterations";
static const std::string KMEANS_CLUSTER_PURGE_EMPTY =
"proxima.kmeans.cluster.purge_empty";
static const std::string KMEANS_CLUSTER_BATCH = "proxima.kmeans.cluster.batch";
"zvec.kmeans.cluster.purge_empty";
static const std::string KMEANS_CLUSTER_BATCH = "zvec.kmeans.cluster.batch";
static const std::string KMEANS_CLUSTER_SEEKER_CLASS =
"proxima.kmeans.cluster.seeker_class";
"zvec.kmeans.cluster.seeker_class";
static const std::string KMEANS_CLUSTER_SEEKER_PARAMS =
"proxima.kmeans.cluster.seeker_params";
"zvec.kmeans.cluster.seeker_params";
//! Mini Batch K-means
static const std::string MINIBATCHKMEANS_CLUSTER_COUNT =
"proxima.minibatchkmeans.cluster.count";
"zvec.minibatchkmeans.cluster.count";
static const std::string MINIBATCHKMEANS_CLUSTER_SHARD_FACTOR =
"proxima.minibatchkmeans.cluster.shard_factor";
"zvec.minibatchkmeans.cluster.shard_factor";
static const std::string MINIBATCHKMEANS_CLUSTER_EPSILON =
"proxima.minibatchkmeans.cluster.epsilon";
"zvec.minibatchkmeans.cluster.epsilon";
static const std::string MINIBATCHKMEANS_CLUSTER_MAX_ITERATIONS =
"proxima.minibatchkmeans.cluster.max_iterations";
"zvec.minibatchkmeans.cluster.max_iterations";
static const std::string MINIBATCHKMEANS_CLUSTER_PURGE_EMPTY =
"proxima.minibatchkmeans.cluster.purge_empty";
"zvec.minibatchkmeans.cluster.purge_empty";
static const std::string MINIBATCHKMEANS_CLUSTER_TRY_COUNT =
"proxima.minibatchkmeans.cluster.try_count";
"zvec.minibatchkmeans.cluster.try_count";
static const std::string MINIBATCHKMEANS_CLUSTER_BATCH_COUNT =
"proxima.minibatchkmeans.cluster.batch_count";
"zvec.minibatchkmeans.cluster.batch_count";
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_CLASS =
"proxima.minibatchkmeans.cluster.seeker_class";
"zvec.minibatchkmeans.cluster.seeker_class";
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_PARAMS =
"proxima.minibatchkmeans.cluster.seeker_params";
"zvec.minibatchkmeans.cluster.seeker_params";
//! K-means++
static const std::string KMEANSPP_CLUSTER_COUNT =
"proxima.kmeanspp.cluster.count";
static const std::string KMEANSPP_CLUSTER_COUNT = "zvec.kmeanspp.cluster.count";
static const std::string KMEANSPP_CLUSTER_SHARD_FACTOR =
"proxima.kmeanspp.cluster.shard_factor";
static const std::string KMEANSPP_CLUSTER_CLASS =
"proxima.kmeanspp.cluster.class";
"zvec.kmeanspp.cluster.shard_factor";
static const std::string KMEANSPP_CLUSTER_CLASS = "zvec.kmeanspp.cluster.class";
static const std::string KMEANSPP_CLUSTER_PARAMS =
"proxima.kmeanspp.cluster.params";
"zvec.kmeanspp.cluster.params";
//! K-MC2
static const std::string KMC2_CLUSTER_COUNT = "proxima.kmc2.cluster.count";
static const std::string KMC2_CLUSTER_COUNT = "zvec.kmc2.cluster.count";
static const std::string KMC2_CLUSTER_SHARD_FACTOR =
"proxima.kmc2.cluster.shard_factor";
"zvec.kmc2.cluster.shard_factor";
static const std::string KMC2_CLUSTER_MARKOV_CHAIN_LENGTH =
"proxima.kmc2.cluster.markov_chain_length";
"zvec.kmc2.cluster.markov_chain_length";
static const std::string KMC2_CLUSTER_ASSUMPTION_FREE =
"proxima.kmc2.cluster.assumption_free";
static const std::string KMC2_CLUSTER_CLASS = "proxima.kmc2.cluster.class";
static const std::string KMC2_CLUSTER_PARAMS = "proxima.kmc2.cluster.params";
"zvec.kmc2.cluster.assumption_free";
static const std::string KMC2_CLUSTER_CLASS = "zvec.kmc2.cluster.class";
static const std::string KMC2_CLUSTER_PARAMS = "zvec.kmc2.cluster.params";
//! Bisecting K-means
static const std::string BIKMEANS_CLUSTER_COUNT =
"proxima.bikmeans.cluster.count";
static const std::string BIKMEANS_CLUSTER_COUNT = "zvec.bikmeans.cluster.count";
static const std::string BIKMEANS_CLUSTER_INIT_COUNT =
"proxima.bikmeans.cluster.init_count";
"zvec.bikmeans.cluster.init_count";
static const std::string BIKMEANS_CLUSTER_PURGE_EMPTY =
"proxima.bikmeans.cluster.purge_empty";
"zvec.bikmeans.cluster.purge_empty";
static const std::string BIKMEANS_CLUSTER_FIRST_CLASS =
"proxima.bikmeans.cluster.first_class";
"zvec.bikmeans.cluster.first_class";
static const std::string BIKMEANS_CLUSTER_SECOND_CLASS =
"proxima.bikmeans.cluster.second_class";
"zvec.bikmeans.cluster.second_class";
static const std::string BIKMEANS_CLUSTER_FIRST_PARAMS =
"proxima.bikmeans.cluster.first_params";
"zvec.bikmeans.cluster.first_params";
static const std::string BIKMEANS_CLUSTER_SECOND_PARAMS =
"proxima.bikmeans.cluster.second_params";
"zvec.bikmeans.cluster.second_params";
//! K-medoids
static const std::string KMEDOIDS_CLUSTER_COUNT =
"proxima.kmedoids.cluster.count";
static const std::string KMEDOIDS_CLUSTER_COUNT = "zvec.kmedoids.cluster.count";
static const std::string KMEDOIDS_CLUSTER_SHARD_FACTOR =
"proxima.kmedoids.cluster.shard_factor";
"zvec.kmedoids.cluster.shard_factor";
static const std::string KMEDOIDS_CLUSTER_EPSILON =
"proxima.kmedoids.cluster.epsilon";
"zvec.kmedoids.cluster.epsilon";
static const std::string KMEDOIDS_CLUSTER_MAX_ITERATIONS =
"proxima.kmedoids.cluster.max_iterations";
"zvec.kmedoids.cluster.max_iterations";
static const std::string KMEDOIDS_CLUSTER_PURGE_EMPTY =
"proxima.kmedoids.cluster.purge_empty";
"zvec.kmedoids.cluster.purge_empty";
static const std::string KMEDOIDS_CLUSTER_BENCH_RATIO =
"proxima.kmedoids.cluster.bench_ratio";
"zvec.kmedoids.cluster.bench_ratio";
static const std::string KMEDOIDS_CLUSTER_ONLY_MEANS =
"proxima.kmedoids.cluster.only_means";
"zvec.kmedoids.cluster.only_means";
static const std::string KMEDOIDS_CLUSTER_WITHOUT_MEANS =
"proxima.kmedoids.cluster.without_means";
"zvec.kmedoids.cluster.without_means";
static const std::string KMEDOIDS_CLUSTER_SEEKER_CLASS =
"proxima.kmedoids.cluster.seeker_class";
"zvec.kmedoids.cluster.seeker_class";
static const std::string KMEDOIDS_CLUSTER_SEEKER_PARAMS =
"proxima.kmedoids.cluster.seeker_params";
"zvec.kmedoids.cluster.seeker_params";
//! Stratified
static const std::string STRATIFIED_CLUSTER_COUNT =
"proxima.stratified.cluster.count";
"zvec.stratified.cluster.count";
static const std::string STRATIFIED_CLUSTER_FIRST_CLASS =
"proxima.stratified.cluster.first_class";
"zvec.stratified.cluster.first_class";
static const std::string STRATIFIED_CLUSTER_SECOND_CLASS =
"proxima.stratified.cluster.second_class";
"zvec.stratified.cluster.second_class";
static const std::string STRATIFIED_CLUSTER_FIRST_COUNT =
"proxima.stratified.cluster.first_count";
"zvec.stratified.cluster.first_count";
static const std::string STRATIFIED_CLUSTER_SECOND_COUNT =
"proxima.stratified.cluster.second_count";
"zvec.stratified.cluster.second_count";
static const std::string STRATIFIED_CLUSTER_FIRST_PARAMS =
"proxima.stratified.cluster.first_params";
"zvec.stratified.cluster.first_params";
static const std::string STRATIFIED_CLUSTER_SECOND_PARAMS =
"proxima.stratified.cluster.second_params";
"zvec.stratified.cluster.second_params";
static const std::string STRATIFIED_CLUSTER_AUTO_TUNING =
"proxima.stratified.cluster.auto_tuning";
"zvec.stratified.cluster.auto_tuning";
static const std::string STRATIFIED_CLUSTER_SECOND_POOL_COUNT =
"proxima.stratified.cluster.second_pool_count";
"zvec.stratified.cluster.second_pool_count";
//! Gap Statistics
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN =
"proxima.gapstats.cluster_estimater.k_min";
"zvec.gapstats.cluster_estimater.k_min";
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX =
"proxima.gapstats.cluster_estimater.k_max";
"zvec.gapstats.cluster_estimater.k_max";
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN_STEP =
"proxima.gapstats.cluster_estimater.k_min_step";
"zvec.gapstats.cluster_estimater.k_min_step";
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX_STEP =
"proxima.gapstats.cluster_estimater.k_max_step";
"zvec.gapstats.cluster_estimater.k_max_step";
static const std::string GAPSTATS_CLUSTER_ESTIMATER_TRY_COUNT =
"proxima.gapstats.cluster_estimater.try_count";
"zvec.gapstats.cluster_estimater.try_count";
static const std::string GAPSTATS_CLUSTER_ESTIMATER_SHARD_FACTOR =
"proxima.gapstats.cluster_estimater.shard_factor";
"zvec.gapstats.cluster_estimater.shard_factor";
static const std::string GAPSTATS_CLUSTER_ESTIMATER_ENABLE_MC2 =
"proxima.gapstats.cluster_estimater.enable_mc2";
"zvec.gapstats.cluster_estimater.enable_mc2";
static const std::string GAPSTATS_CLUSTER_ESTIMATER_MARKOV_CHAIN_LENGTH =
"proxima.gapstats.cluster_estimater.markov_chain_length";
"zvec.gapstats.cluster_estimater.markov_chain_length";
static const std::string GAPSTATS_CLUSTER_ESTIMATER_CLUSTER_CLASS =
"proxima.gapstats.cluster_estimater.cluster_class";
"zvec.gapstats.cluster_estimater.cluster_class";
static const std::string CLUSTER_TRAINER_SAMPLE_COUNT =
"proxima.cluster.trainer.sample_count";
"zvec.cluster.trainer.sample_count";
static const std::string CLUSTER_TRAINER_SAMPLE_RATIO =
"proxima.cluster.trainer.sample_ratio";
"zvec.cluster.trainer.sample_ratio";
static const std::string CLUSTER_TRAINER_THREAD_COUNT =
"proxima.cluster.trainer.thread_count";
"zvec.cluster.trainer.thread_count";
static const std::string CLUSTER_TRAINER_FILE_NAME =
"proxima.cluster.trainer.file_name";
"zvec.cluster.trainer.file_name";
static const std::string CLUSTER_TRAINER_CLASS_NAME =
"proxima.cluster.trainer.class_name";
"zvec.cluster.trainer.class_name";
static const std::string STRATIFIED_TRAINER_SAMPLE_COUNT =
"proxima.stratified.trainer.sample_count";
"zvec.stratified.trainer.sample_count";
static const std::string STRATIFIED_TRAINER_SAMPLE_RATIO =
"proxima.stratified.trainer.sample_ratio";
"zvec.stratified.trainer.sample_ratio";
static const std::string STRATIFIED_TRAINER_THREAD_COUNT =
"proxima.stratified.trainer.thread_count";
"zvec.stratified.trainer.thread_count";
static const std::string STRATIFIED_TRAINER_FILE_NAME =
"proxima.stratified.trainer.file_name";
"zvec.stratified.trainer.file_name";
static const std::string STRATIFIED_TRAINER_CLASS_NAME =
"proxima.stratified.trainer.class_name";
"zvec.stratified.trainer.class_name";
static const std::string STRATIFIED_TRAINER_CLUSTER_COUNT =
"proxima.stratified.trainer.cluster_count";
"zvec.stratified.trainer.cluster_count";
static const std::string STRATIFIED_TRAINER_AUTOAUNE =
"proxima.stratified.trainer.autotune";
"zvec.stratified.trainer.autotune";
static const std::string STRATIFIED_TRAINER_PARAMS_IN_LEVEL_PREFIX =
"proxima.stratified.trainer.cluster_params_in_level_";
"zvec.stratified.trainer.cluster_params_in_level_";
static const std::string MULTI_CHUNK_CLUSTER_COUNT =
"zvec.cluster.multi_chunk_cluster.count";
static const std::string MULTI_CHUNK_CLUSTER_CHUNK_COUNT =
"zvec.cluster.multi_chunk_cluster.chunk_count";
static const std::string MULTI_CHUNK_CLUSTER_THREAD_COUNT =
"zvec.cluster.multi_chunk_cluster.thread_count";
static const std::string MULTI_CHUNK_CLUSTER_EPSILON =
"zvec.cluster.multi_chunk_cluster.epsilon";
static const std::string MULTI_CHUNK_CLUSTER_MAX_ITERATIONS =
"zvec.cluster.multi_chunk_cluster.max_iterations";
static const std::string MULTI_CHUNK_CLUSTER_MARKOV_CHAIN_LENGTH =
"zvec.cluster.multi_chunk_cluster.markov_chain_length";
} // namespace core
} // namespace zvec
@@ -0,0 +1,436 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "multi_chunk_cluster.h"
#include <chrono>
namespace zvec {
namespace core {
bool MultiChunkClusterAlgorithm::is_valid(void) const {
return features_ && features_->count();
}
bool MultiChunkClusterAlgorithm::check_centroids(
const IndexCluster::CentroidList & /*cents*/) {
return true;
}
void MultiChunkClusterAlgorithm::init_centroids(
size_t count, IndexCluster::CentroidList *out) {
// Just resize, because the get random centroid step is done by cluster_once
out->resize(count);
}
int MultiChunkClusterAlgorithm::reset(void) {
features_.reset();
return 0;
}
int MultiChunkClusterAlgorithm::cleanup(void) {
features_.reset();
return 0;
}
void MultiChunkClusterAlgorithm::suggest(uint32_t k) {
cluster_count_ = k;
}
int MultiChunkClusterAlgorithm::update(const ailego::Params &params) {
this->update_params(params);
return 0;
}
//! MultiChunkCluster
int MultiChunkClusterAlgorithm::update_params(const ailego::Params &params) {
params.get(GENERAL_THREAD_COUNT, &thread_count_);
params.get(GENERAL_CLUSTER_COUNT, &cluster_count_);
params.get(MULTI_CHUNK_CLUSTER_THREAD_COUNT, &thread_count_);
params.get(MULTI_CHUNK_CLUSTER_COUNT, &cluster_count_);
params.get(MULTI_CHUNK_CLUSTER_CHUNK_COUNT, &chunk_count_);
params.get(MULTI_CHUNK_CLUSTER_EPSILON, &epsilon_);
params.get(MULTI_CHUNK_CLUSTER_MAX_ITERATIONS, &max_iterations_);
params.get(MULTI_CHUNK_CLUSTER_MARKOV_CHAIN_LENGTH, &markov_chain_length_);
return 0;
}
int MultiChunkClusterAlgorithm::init_distance_func() {
IndexMetric::Pointer metric_{};
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!metric_) {
LOG_ERROR("Create metric %s failed.", meta_.metric_name().c_str());
return IndexError_Unsupported;
}
int ret = metric_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("IndexMetric init failed wit ret %d.", ret);
return ret;
}
distance_func_ = metric_->distance_matrix(1, 1);
if (!distance_func_) {
LOG_ERROR("DistanceMatrix function is nullptr.");
return IndexError_Unsupported;
}
return 0;
}
int MultiChunkClusterAlgorithm::do_chunk() {
if (chunk_count_ == 0) {
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
return IndexError_InvalidArgument;
}
size_t large_chunk_count = meta_.dimension() % chunk_count_;
size_t base_chunk_dim_ = meta_.dimension() / chunk_count_;
chunk_dims_.clear();
for (size_t i = 0; i < chunk_count_; ++i) {
if (i < large_chunk_count) {
chunk_dims_.push_back(base_chunk_dim_ + 1);
} else {
chunk_dims_.push_back(base_chunk_dim_);
}
}
chunk_dim_offsets_.clear();
chunk_dim_offsets_.push_back(0);
for (size_t i = 1; i < chunk_count_; ++i) {
chunk_dim_offsets_.push_back(chunk_dim_offsets_[i - 1] +
chunk_dims_[i - 1]);
}
chunk_dim_offsets_.push_back(meta_.dimension());
return 0;
}
int MultiChunkClusterAlgorithm::init(const IndexMeta &meta,
const ailego::Params &params) {
meta_ = meta;
int ret = update_params(params);
if (ret != 0) {
return ret;
}
ret = do_chunk();
if (ret != 0) {
return ret;
}
ret = init_distance_func();
if (ret != 0) {
return ret;
}
return 0;
}
int MultiChunkClusterAlgorithm::mount(IndexFeatures::Pointer features) {
if (!features) {
return IndexError_InvalidArgument;
}
if (!features->is_matched(meta_)) {
return IndexError_Mismatch;
}
auto data_type = meta_.data_type();
if (data_type != IndexMeta::DataType::DT_FP32 &&
data_type != IndexMeta::DataType::DT_FP16) {
LOG_ERROR("Unsupported meta type %u", data_type);
return IndexError_Unsupported;
}
features_ = std::move(features);
return 0;
}
//! cluster
int MultiChunkClusterAlgorithm::cluster(IndexThreads::Pointer threads,
IndexCluster::CentroidList &cents) {
if (chunk_count_ == 0) {
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
return IndexError_InvalidArgument;
}
if (cluster_count_ == 0) {
LOG_ERROR("Invalid cluster Count: %u", cluster_count_);
return IndexError_InvalidArgument;
}
if (!threads) {
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
}
auto task_group = threads->make_group();
if (!task_group) {
LOG_ERROR("Failed to create task group");
return IndexError_Runtime;
}
cents.clear();
cents.resize(chunk_count_ * cluster_count_);
std::atomic<size_t> finished{0};
for (size_t i = 0; i < threads->count(); ++i) {
task_group->submit(
ailego::Closure::New(this, &MultiChunkClusterAlgorithm::do_cluster, i,
threads->count(), &cents, &finished));
}
while (!task_group->is_finished()) {
std::unique_lock<std::mutex> lk(mutex_);
cond_.wait_until(lk, std::chrono::system_clock::now() +
std::chrono::seconds(check_interval_secs_));
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to cluster while waiting finish");
return errcode_;
}
LOG_INFO("Finish Chunk Count %zu, Finished Percent %.3f%%", finished.load(),
finished.load() * 100.0f / chunk_count_);
}
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to cluster while waiting finish");
return errcode_;
}
task_group->wait_finish();
return 0;
}
//! Classify
int MultiChunkClusterAlgorithm::classify(
IndexThreads::Pointer /*threads*/, IndexCluster::CentroidList & /*cents*/) {
return IndexError_Unsupported;
}
//! Label
int MultiChunkClusterAlgorithm::label(IndexThreads::Pointer threads,
const IndexCluster::CentroidList &cents,
std::vector<uint32_t> *out) {
if (chunk_count_ == 0) {
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
return IndexError_InvalidArgument;
}
if (cents.empty()) {
LOG_ERROR("The input centroid's list is empty.");
return IndexError_InvalidArgument;
}
if (!this->check_centroids(cents)) {
LOG_ERROR("The input centroid's list includes some invalid centroids.");
return IndexError_InvalidArgument;
}
if (!this->is_valid()) {
LOG_ERROR("The cluster is not ready.");
return IndexError_NoReady;
}
if (cluster_count_ == 0) {
LOG_ERROR("Invalid cluster Count: %u", cluster_count_);
return IndexError_InvalidArgument;
}
if (!threads) {
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
if (!threads) {
return IndexError_NoMemory;
}
}
auto task_group = threads->make_group();
if (!task_group) {
LOG_ERROR("Failed to create task group");
return IndexError_Runtime;
}
size_t features_count = features_->count();
out->resize(features_count * chunk_count_);
std::atomic<size_t> finished{0};
for (size_t i = 0; i < threads->count(); ++i) {
task_group->submit(
ailego::Closure::New(this, &MultiChunkClusterAlgorithm::do_label, i,
threads->count(), cents, out, &finished));
}
while (!task_group->is_finished()) {
std::unique_lock<std::mutex> lk(mutex_);
cond_.wait_until(lk, std::chrono::system_clock::now() +
std::chrono::seconds(check_interval_secs_));
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to cluster while waiting finish");
return errcode_;
}
LOG_INFO("Finish label cnt %zu, finished percent %.3f%%", finished.load(),
finished.load() * 100.0f / features_count);
}
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to cluster while waiting finish");
return errcode_;
}
task_group->wait_finish();
return 0;
}
//! Cluster
int MultiChunkCluster::cluster(IndexThreads::Pointer threads,
IndexCluster::CentroidList &cents) {
return algorithm_->cluster(std::move(threads), cents);
}
//! Classify
int MultiChunkCluster::classify(IndexThreads::Pointer threads,
IndexCluster::CentroidList &cents) {
return algorithm_->classify(std::move(threads), cents);
}
//! Label
int MultiChunkCluster::label(IndexThreads::Pointer threads,
const IndexCluster::CentroidList &cents,
std::vector<uint32_t> *out) {
return algorithm_->label(std::move(threads), cents, out);
}
//! Update Cluster
int MultiChunkCluster::update(const ailego::Params &params) {
return algorithm_->update(params);
}
//! Reset Cluster
int MultiChunkCluster::reset(void) {
return algorithm_->reset();
}
//! Cleanup Cluster
int MultiChunkCluster::cleanup(void) {
return algorithm_->cleanup();
}
//! Suggest dividing to K clusters
void MultiChunkCluster::suggest(uint32_t k) {
algorithm_->suggest(k);
}
int MultiChunkCluster::mount(IndexFeatures::Pointer feats) {
return algorithm_->mount(feats);
}
int MultiChunkCluster::init(const IndexMeta &meta,
const ailego::Params &params) {
IndexMeta new_meta(meta.data_type(), meta.dimension());
if (meta.metric_name() == "Cosine") {
new_meta.set_dimension(meta.dimension() - 1);
new_meta.set_metric("InnerProduct", 0, ailego::Params());
}
auto data_type = new_meta.data_type();
if (new_meta.metric_name() == "InnerProduct") {
switch (data_type) {
case IndexMeta::DataType::DT_FP16: {
algorithm_.reset(
new (std::nothrow)
MultiChunkNumericalInnerProductAlgorithm<ailego::Float16>);
break;
}
case IndexMeta::DataType::DT_FP32: {
algorithm_.reset(new (std::nothrow)
MultiChunkNumericalInnerProductAlgorithm<float>);
break;
}
case IndexMeta::DataType::DT_FP64: {
algorithm_.reset(new (std::nothrow)
MultiChunkNumericalInnerProductAlgorithm<double>);
break;
}
case IndexMeta::DataType::DT_INT8: {
algorithm_.reset(new (std::nothrow)
MultiChunkNumericalInnerProductAlgorithm<int8_t>);
break;
}
case IndexMeta::DataType::DT_INT16: {
algorithm_.reset(new (std::nothrow)
MultiChunkNumericalInnerProductAlgorithm<int16_t>);
break;
}
default: {
LOG_ERROR("Unsupported feature types %d.", data_type);
return IndexError_Mismatch;
}
}
} else {
switch (data_type) {
case IndexMeta::DataType::DT_FP16: {
algorithm_.reset(new (std::nothrow)
MultiChunkNumericalAlgorithm<ailego::Float16>);
break;
}
case IndexMeta::DataType::DT_FP32: {
algorithm_.reset(new (std::nothrow)
MultiChunkNumericalAlgorithm<float>);
break;
}
case IndexMeta::DataType::DT_FP64: {
algorithm_.reset(new (std::nothrow)
MultiChunkNumericalAlgorithm<double>);
break;
}
case IndexMeta::DataType::DT_INT8: {
algorithm_.reset(new (std::nothrow)
MultiChunkNumericalAlgorithm<int8_t>);
break;
}
case IndexMeta::DataType::DT_INT16: {
algorithm_.reset(new (std::nothrow)
MultiChunkNumericalAlgorithm<int16_t>);
break;
}
default: {
LOG_ERROR("Unsupported feature types %d.", data_type);
return IndexError_Mismatch;
}
}
}
algorithm_->init(new_meta, params);
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,466 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/algorithm/kmeans.h>
#include <ailego/parallel/multi_thread_list.h>
#include <zvec/ailego/internal/platform.h>
#include <zvec/core/framework/index_cluster.h>
#include <zvec/core/framework/index_error.h>
#include <zvec/core/framework/index_factory.h>
#include "cluster_params.h"
namespace zvec {
namespace core {
//! MultiChunkClusterAlgorithm
class MultiChunkClusterAlgorithm {
public:
typedef std::shared_ptr<MultiChunkClusterAlgorithm> Pointer;
//! Constructor
MultiChunkClusterAlgorithm() = default;
//! Destructor
virtual ~MultiChunkClusterAlgorithm() = default;
//! Initialize Cluster
int init(const IndexMeta &meta, const ailego::Params &params);
//! Cleanup Cluster
int cleanup();
//! Reset Cluster
int reset();
//! Update Cluster
int update(const ailego::Params &params);
//! Suggest dividing to K clusters
void suggest(uint32_t k);
//! Mount features
int mount(IndexFeatures::Pointer feats);
//! Cluster
int cluster(IndexThreads::Pointer threads, IndexCluster::CentroidList &cents);
//! Classify
int classify(IndexThreads::Pointer threads,
IndexCluster::CentroidList &cents);
//! Label
int label(IndexThreads::Pointer threads,
const IndexCluster::CentroidList &cents,
std::vector<uint32_t> *out);
const std::vector<uint32_t> &chunk_dims() const {
return chunk_dims_;
}
const std::vector<uint32_t> &chunk_dim_offsets() const {
return chunk_dim_offsets_;
}
protected:
//! Check Centroids
bool check_centroids(const IndexCluster::CentroidList &cents);
//! Test if it is valid
bool is_valid() const;
//! Do chunk
int do_chunk();
//! Update parameters
int update_params(const ailego::Params &params);
int init_distance_func();
//! cluster thread
virtual void do_cluster(size_t idx, size_t chunk_step,
IndexCluster::CentroidList *cents,
std::atomic<size_t> *finished) = 0;
//! label thread
virtual void do_label(size_t idx, size_t step,
const IndexCluster::CentroidList &cents,
std::vector<uint32_t> *out,
std::atomic<size_t> *finished) = 0;
//! Initialize Centroids
void init_centroids(size_t count, IndexCluster::CentroidList *out);
private:
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
protected:
uint32_t cluster_count_{0u};
uint32_t thread_count_{0u};
uint32_t chunk_count_{0u};
uint32_t max_iterations_{20u};
bool assumption_free_{false};
uint32_t markov_chain_length_{32};
double epsilon_{std::numeric_limits<float>::epsilon()};
int errcode_{0};
std::atomic_bool error_{false};
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
std::mutex mutex_{};
std::condition_variable cond_{};
IndexMeta meta_{};
IndexFeatures::Pointer features_{};
std::vector<uint32_t> chunk_dims_;
std::vector<uint32_t> chunk_dim_offsets_;
IndexMetric::MatrixDistance distance_func_{nullptr};
};
/*! Numerical cluster algorithm
*/
template <typename T>
class MultiChunkNumericalAlgorithm : public MultiChunkClusterAlgorithm {
public:
//! Type of value
using ValueType = typename std::remove_cv<T>::type;
// Check supporting type
static_assert(ailego::IsArithmetic<ValueType>::value,
"ValueType must be arithmetic");
//! Constructor
MultiChunkNumericalAlgorithm() = default;
//! Destructor
~MultiChunkNumericalAlgorithm() = default;
protected:
//! cluster thread
void do_cluster(size_t idx, size_t chunk_step,
IndexCluster::CentroidList *cents,
std::atomic<size_t> *finished);
//! label thread
void do_label(size_t idx, size_t step,
const IndexCluster::CentroidList &cents,
std::vector<uint32_t> *out, std::atomic<size_t> *finished);
};
//! cluster thread
template <typename T>
void MultiChunkNumericalAlgorithm<T>::do_cluster(
size_t idx, size_t chunk_step, IndexCluster::CentroidList *cents,
std::atomic<size_t> *finished) {
for (size_t chunk = idx; chunk < chunk_count_; chunk += chunk_step) {
auto chunk_dim = chunk_dims_[chunk];
ailego::NumericalKmeans<T, IndexThreads> algorithm(cluster_count_,
chunk_dim);
// mount features into algorithm
auto features_count = features_->count();
algorithm.feature_matrix_reserve(features_count);
for (size_t i = 0; i < features_count; ++i) {
auto vec = reinterpret_cast<const T *>(features_->element(i));
algorithm.append(vec + chunk_dim_offsets_[chunk], chunk_dim);
}
IndexThreads::Pointer local_threads =
std::make_shared<SingleQueueIndexThreads>(1, false);
if (!local_threads) {
error_ = IndexError_NoMemory;
return;
}
ailego::Kmc2CentroidsGenerator<ailego::NumericalKmeans<T, IndexThreads>,
IndexThreads>
cent_gen;
cent_gen.set_chain_length(markov_chain_length_);
cent_gen.set_assumption_free(assumption_free_);
cent_gen(&algorithm, *local_threads);
double cost = 0.0;
for (uint32_t i = 0; i < max_iterations_; ++i) {
double old_cost, new_epsilon;
old_cost = cost;
bool result = algorithm.cluster_once(*local_threads, &cost);
if (result != true) {
LOG_ERROR("(%u) Failed to cluster.", i + 1);
errcode_ = -1;
return;
}
new_epsilon = std::abs(cost - old_cost);
if (new_epsilon < epsilon_) {
break;
}
}
auto &chunk_cents = algorithm.centroids();
for (size_t i = 0; i < chunk_cents.count(); ++i) {
size_t global_cent_idx = chunk * cluster_count_ + i;
IndexCluster::Centroid *centroid = &(cents->at(global_cent_idx));
centroid->set_score(algorithm.context().clusters()[i].cost());
centroid->set_follows(algorithm.context().clusters()[i].count());
centroid->set_feature(algorithm.centroids()[i],
chunk_dim * meta_.unit_size());
}
LOG_INFO("(%zu) Chunk Done. Clusters Count: %zu, Features: %zu, Cost: %f",
chunk, algorithm.centroids().count(), features_->count(), cost);
(*finished)++;
}
return;
}
//! label thread
template <typename T>
void MultiChunkNumericalAlgorithm<T>::do_label(
size_t idx, size_t step, const IndexCluster::CentroidList &cents,
std::vector<uint32_t> *out, std::atomic<size_t> *finished) {
for (size_t id = idx; id < features_->count(); id += step) {
const T *feat = reinterpret_cast<const T *>(features_->element(id));
for (size_t chunk = 0; chunk < chunk_count_; ++chunk) {
size_t chunk_dim_offset = chunk_dim_offsets_[chunk];
size_t chunk_dim = chunk_dims_[chunk];
uint32_t sel_index = 0;
float sel_score = std::numeric_limits<float>::max();
for (uint32_t cluster = 0; cluster < cluster_count_; ++cluster) {
float score{0.0};
distance_func_(cents[chunk * cluster_count_ + cluster].feature(),
feat + chunk_dim_offset, chunk_dim, &score);
if (score < sel_score) {
sel_score = score;
sel_index = cluster;
}
}
(*out)[id * chunk_count_ + chunk] = static_cast<uint32_t>(sel_index);
}
(*finished)++;
}
}
/*! Inner Product Cluster Algorithm
*/
template <typename T>
class MultiChunkNumericalInnerProductAlgorithm
: public MultiChunkClusterAlgorithm {
public:
//! Type of value
using ValueType = typename std::remove_cv<T>::type;
// Check supporting type
static_assert(ailego::IsArithmetic<ValueType>::value,
"ValueType must be arithmetic");
//! Constructor
MultiChunkNumericalInnerProductAlgorithm() = default;
//! Destructor
~MultiChunkNumericalInnerProductAlgorithm() = default;
protected:
//! cluster thread
void do_cluster(size_t idx, size_t chunk_step,
IndexCluster::CentroidList *cents,
std::atomic<size_t> *finished);
//! label thread
void do_label(size_t idx, size_t chunk_step,
const IndexCluster::CentroidList &cents,
std::vector<uint32_t> *out, std::atomic<size_t> *finished);
};
//! cluster thread
template <typename T>
void MultiChunkNumericalInnerProductAlgorithm<T>::do_cluster(
size_t idx, size_t chunk_step, IndexCluster::CentroidList *cents,
std::atomic<size_t> *finished) {
for (size_t chunk = idx; chunk < chunk_count_; chunk += chunk_step) {
auto chunk_dim = chunk_dims_[chunk];
ailego::NumericalInnerProductKmeans<T, IndexThreads> algorithm(
cluster_count_, chunk_dim);
// mount features into algorithm
auto features_count = features_->count();
algorithm.feature_matrix_reserve(features_count);
for (size_t i = 0; i < features_count; ++i) {
auto vec = reinterpret_cast<const T *>(features_->element(i));
algorithm.append(vec + chunk_dim_offsets_[chunk], chunk_dim);
}
IndexThreads::Pointer local_threads =
std::make_shared<SingleQueueIndexThreads>(1, false);
if (!local_threads) {
error_ = IndexError_NoMemory;
return;
}
ailego::Kmc2CentroidsGenerator<
ailego::NumericalInnerProductKmeans<T, IndexThreads>, IndexThreads>
cent_gen;
cent_gen.set_chain_length(markov_chain_length_);
cent_gen.set_assumption_free(assumption_free_);
cent_gen(&algorithm, *local_threads);
double cost = 0.0;
for (uint32_t i = 0; i < max_iterations_; ++i) {
double old_cost, new_epsilon;
old_cost = cost;
bool result = algorithm.cluster_once(*local_threads, &cost);
if (result != true) {
LOG_ERROR("(%zu) Failed to cluster.", (size_t)(i + 1));
errcode_ = -1;
return;
}
new_epsilon = std::abs(cost - old_cost);
if (new_epsilon < epsilon_) {
break;
}
}
auto &chunk_cents = algorithm.centroids();
for (size_t i = 0; i < chunk_cents.count(); ++i) {
size_t global_cent_idx = chunk * cluster_count_ + i;
IndexCluster::Centroid *centroid = &(cents->at(global_cent_idx));
centroid->set_score(algorithm.context().clusters()[i].cost());
centroid->set_follows(algorithm.context().clusters()[i].count());
centroid->set_feature(algorithm.centroids()[i],
chunk_dim * meta_.unit_size());
}
LOG_INFO("(%zu) Chunk Done. Clusters Count: %zu, Features: %zu, Cost: %f",
chunk, algorithm.centroids().count(), features_->count(), cost);
(*finished)++;
}
}
//! label thread
template <typename T>
void MultiChunkNumericalInnerProductAlgorithm<T>::do_label(
size_t idx, size_t step, const IndexCluster::CentroidList &cents,
std::vector<uint32_t> *out, std::atomic<size_t> *finished) {
for (size_t id = idx; id < features_->count(); id += step) {
const T *feat = reinterpret_cast<const T *>(features_->element(id));
for (size_t chunk = 0; chunk < chunk_count_; ++chunk) {
size_t chunk_dim_offset = chunk_dim_offsets_[chunk];
size_t chunk_dim = chunk_dims_[chunk];
uint32_t sel_index = 0;
float sel_score = std::numeric_limits<float>::max();
for (uint32_t cluster = 0; cluster < cluster_count_; ++cluster) {
float score{0.0};
distance_func_(cents[chunk * cluster_count_ + cluster].feature(),
feat + chunk_dim_offset, chunk_dim, &score);
if (score < sel_score) {
sel_score = score;
sel_index = cluster;
}
}
(*out)[id * chunk_count_ + chunk] = sel_index;
}
(*finished)++;
}
}
//! MultiChunkCluster
class MultiChunkCluster {
public:
std::shared_ptr<MultiChunkCluster> Pointer;
//! Constructor
MultiChunkCluster() = default;
//! Destructor
~MultiChunkCluster() = default;
//! Initialize Cluster
int init(const IndexMeta &meta, const ailego::Params &params);
//! Cleanup Cluster
int cleanup();
//! Reset Cluster
int reset();
//! Update Cluster
int update(const ailego::Params &params);
//! Suggest dividing to K clusters
void suggest(uint32_t k);
//! Mount features
int mount(IndexFeatures::Pointer feats);
//! Cluster
int cluster(IndexThreads::Pointer threads, IndexCluster::CentroidList &cents);
//! Classify
int classify(IndexThreads::Pointer threads,
IndexCluster::CentroidList &cents);
//! Label
int label(IndexThreads::Pointer threads,
const IndexCluster::CentroidList &cents,
std::vector<uint32_t> *out);
const std::vector<uint32_t> &chunk_dims() const {
return algorithm_->chunk_dims();
}
const std::vector<uint32_t> &chunk_dim_offsets() const {
return algorithm_->chunk_dim_offsets();
}
protected:
MultiChunkClusterAlgorithm::Pointer algorithm_{};
};
} // namespace core
} // namespace zvec
+57
View File
@@ -0,0 +1,57 @@
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
file(GLOB_RECURSE ALL_SRCS *.cc *.c)
# The DiskAnn plugin is loaded at runtime via zvec::LoadDiskAnnPlugin() with
# RTLD_GLOBAL, so its undefined references to internal zvec symbols
# (core_framework, core_knn_cluster, zvec_ailego, ...) are resolved at load
# time against the hosting binary (_zvec.so for the Python extension, the
# test executable for gtest, or libzvec_core for tools).
#
# As a consequence the plugin .so must NOT carry NEEDED entries for those
# libs: otherwise the dynamic loader would try to resolve them from disk
# before invoking dlopen's RTLD_GLOBAL symbol sharing, and the Python wheel
# (which only ships _zvec.so + the plugin) would fail to load.
#
# We therefore link the plugin only against its system-level dependency
# (libaio) and rely on the global include dirs plus -Wl,--unresolved-symbols
# =ignore-all to let the linker build a shared library with unresolved
# references to internal APIs.
set(CORE_KNN_DISKANN_LIBS "")
if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386")
list(APPEND CORE_KNN_DISKANN_LIBS aio)
endif()
cc_library(
NAME core_knn_diskann
SHARED STRICT
SRCS *.cc
LIBS ${CORE_KNN_DISKANN_LIBS}
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
VERSION "${PROXIMA_ZVEC_VERSION}"
)
# Internal zvec libs are referenced only for header availability; the actual
# symbol resolution happens at dlopen time. Expose their public include dirs
# without adding them as link dependencies.
foreach(_dep zvec_ailego core_framework core_knn_cluster)
if(TARGET ${_dep})
target_include_directories(core_knn_diskann PRIVATE
$<TARGET_PROPERTY:${_dep},INTERFACE_INCLUDE_DIRECTORIES>)
add_dependencies(core_knn_diskann ${_dep})
endif()
endforeach()
# Allow the plugin to have unresolved symbols that will be satisfied at
# dlopen(RTLD_NOW | RTLD_GLOBAL) time by the hosting binary.
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
target_link_options(core_knn_diskann PRIVATE
"LINKER:--unresolved-symbols=ignore-all")
endif()
# Rename the artifact to libzvec_diskann_plugin.so so it is clearly identified
# as an optional plugin that users load at runtime via zvec::LoadDiskAnnPlugin().
set_target_properties(core_knn_diskann PROPERTIES
OUTPUT_NAME zvec_diskann_plugin
)
@@ -0,0 +1,324 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_algorithm.h"
#include <algorithm>
#include <set>
#include <zvec/core/framework/index_holder.h>
#include "diskann_util.h"
namespace zvec {
namespace core {
DiskAnnAlgorithm::DiskAnnAlgorithm(DiskAnnEntity &entity, uint32_t max_degree)
: entity_(entity), max_degree_(max_degree), lock_pool_(kLockCnt) {}
std::vector<diskann_id_t> DiskAnnAlgorithm::get_init_ids(DiskAnnContext *ctx) {
const auto &entity = ctx->get_entity();
std::vector<diskann_id_t> init_ids;
init_ids.emplace_back(entity.medoid());
return init_ids;
}
int DiskAnnAlgorithm::add_node(diskann_id_t id, DiskAnnContext *ctx) {
const void *vec = entity_.get_vector(id);
ctx->reset_query(vec);
std::vector<diskann_id_t> pruned_list;
int ret = search_neighbor_and_prune(id, pruned_list, ctx);
if (ret != 0) {
return ret;
}
{
auto lock = lock_for(id);
entity_.set_neighbors(id, pruned_list);
}
return inter_insert(id, pruned_list, ctx);
}
int DiskAnnAlgorithm::prune_node(diskann_id_t id, DiskAnnContext *ctx) {
DistCalculator &dc = ctx->dist_calculator();
auto neighbors = entity_.get_neighbors(id);
if (neighbors.first > max_degree_) {
std::set<diskann_id_t> dummy_visited;
std::vector<Neighbor> dummy_pool(0);
std::vector<diskann_id_t> new_out_neighbors;
for (size_t i = 0; i < neighbors.first; ++i) {
diskann_id_t node_id = (neighbors.second)[i];
auto itr = dummy_visited.find(node_id);
if (itr == dummy_visited.end() && node_id != id) {
float dist = dc.dist(id, node_id);
dummy_pool.emplace_back(Neighbor(node_id, dist));
dummy_visited.insert(node_id);
}
}
prune_neighbors(id, dummy_pool, new_out_neighbors, ctx);
{
auto lock = lock_for(id);
entity_.set_neighbors(id, new_out_neighbors);
}
}
return 0;
}
int DiskAnnAlgorithm::inter_insert(diskann_id_t id,
std::vector<diskann_id_t> &pruned_list,
DiskAnnContext *ctx) {
DistCalculator &dc = ctx->dist_calculator();
for (auto &des : pruned_list) {
std::vector<diskann_id_t> new_neighbors;
bool need_prune = false;
{
auto lock = lock_for(des);
auto neighbors = entity_.get_neighbors(des);
bool found = false;
for (size_t i = 0; i < neighbors.first; ++i) {
if ((neighbors.second)[i] == id) {
found = true;
break;
}
}
if (!found) {
if (neighbors.first <
static_cast<uint64_t>(DiskAnnEntity::kDefaultGraphSlackFactor *
max_degree_)) {
entity_.add_neighbor(des, id);
need_prune = false;
} else {
new_neighbors.resize(neighbors.first + 1);
memcpy(&new_neighbors[0], neighbors.second,
sizeof(diskann_id_t) * neighbors.first);
new_neighbors[neighbors.first] = id;
need_prune = true;
}
}
}
if (need_prune) {
std::set<diskann_id_t> new_visited;
std::vector<Neighbor> new_pool(0);
size_t reserve_size = static_cast<size_t>(std::ceil(
1.05 * DiskAnnEntity::kDefaultGraphSlackFactor * max_degree_));
new_pool.reserve(reserve_size);
for (auto node_id : new_neighbors) {
if (new_visited.find(node_id) == new_visited.end() && node_id != des) {
float dist = dc.dist(des, node_id);
new_pool.emplace_back(Neighbor(node_id, dist));
new_visited.insert(node_id);
}
}
std::vector<diskann_id_t> new_pruned_neighbors;
prune_neighbors(des, new_pool, new_pruned_neighbors, ctx);
{
auto lock = lock_for(des);
entity_.set_neighbors(des, new_pruned_neighbors);
}
}
}
return 0;
}
int DiskAnnAlgorithm::iterate_to_fixed_point(
const std::vector<diskann_id_t> &init_ids, DiskAnnContext *ctx) {
DistCalculator &dc = ctx->dist_calculator();
std::vector<Neighbor> &expanded_nodes = ctx->expanded_nodes();
NeighborPriorityQueue &best_list_nodes = ctx->best_list_nodes();
VisitFilter &visit = ctx->visit_filter();
best_list_nodes.reserve(ctx->list_size());
for (auto id : init_ids) {
const void *vec = entity_.get_vector(id);
float distance = dc.dist(vec);
Neighbor nn = Neighbor(id, distance);
best_list_nodes.insert(nn);
}
while (best_list_nodes.has_unexpanded_node()) {
auto neighbor = best_list_nodes.closest_unexpanded();
auto node_id = neighbor.id;
expanded_nodes.emplace_back(neighbor);
std::vector<diskann_id_t> id_scratch;
{
auto lock = lock_for(node_id);
auto neighbors = entity_.get_neighbors(node_id);
for (size_t i = 0; i < neighbors.first; ++i) {
diskann_id_t neighbor_id = (neighbors.second)[i];
if (!visit.visited(neighbor_id)) {
id_scratch.push_back(neighbor_id);
visit.set_visited(neighbor_id);
}
}
}
for (size_t i = 0; i < id_scratch.size(); ++i) {
diskann_id_t id = id_scratch[i];
const void *vec = entity_.get_vector(id);
float dist = dc.dist(vec);
best_list_nodes.insert(Neighbor(id, dist));
}
}
return 0;
}
int DiskAnnAlgorithm::occlude_list(diskann_id_t id, std::vector<Neighbor> &pool,
std::vector<diskann_id_t> &result,
DiskAnnContext *ctx) {
if (pool.size() == 0) return 0;
DistCalculator &dc = ctx->dist_calculator();
ailego_assert(std::is_sorted(pool.begin(), pool.end()));
ailego_assert(result.size() == 0);
if (pool.size() > max_candidate_size_) {
pool.resize(max_candidate_size_);
}
std::vector<float> &occlude_factor = ctx->occlude_factor();
occlude_factor.clear();
occlude_factor.insert(occlude_factor.end(), pool.size(), 0.0f);
float cur_alpha = 1;
while (cur_alpha <= alpha_ && result.size() < max_degree_) {
for (auto iter = pool.begin();
result.size() < max_degree_ && iter != pool.end(); ++iter) {
if (occlude_factor[iter - pool.begin()] > cur_alpha) {
continue;
}
occlude_factor[iter - pool.begin()] = std::numeric_limits<float>::max();
if (iter->id != id) {
result.push_back(iter->id);
}
for (auto iter2 = iter + 1; iter2 != pool.end(); iter2++) {
auto t = iter2 - pool.begin();
if (occlude_factor[t] > alpha_) {
continue;
}
float djk = dc.dist(iter2->id, iter->id);
occlude_factor[t] =
(djk == 0) ? std::numeric_limits<float>::max()
: std::max(occlude_factor[t], iter2->distance / djk);
}
}
cur_alpha *= 1.2f;
}
return 0;
}
int DiskAnnAlgorithm::prune_neighbors(diskann_id_t id,
std::vector<Neighbor> &pool,
std::vector<diskann_id_t> &pruned_list,
DiskAnnContext *ctx) {
if (pool.size() == 0) {
pruned_list.clear();
return 0;
}
std::sort(pool.begin(), pool.end());
pruned_list.clear();
pruned_list.reserve(max_degree_);
occlude_list(id, pool, pruned_list, ctx);
ailego_assert(pruned_list.size() <= max_degree_);
if (saturate_graph_ && alpha_ > 1) {
for (const auto &node : pool) {
if (pruned_list.size() >= max_degree_) {
break;
}
if ((std::find(pruned_list.begin(), pruned_list.end(), node.id) ==
pruned_list.end()) &&
node.id != id) {
pruned_list.push_back(node.id);
}
}
}
return 0;
}
int DiskAnnAlgorithm::search_neighbor_and_prune(
diskann_id_t id, std::vector<diskann_id_t> &pruned_list,
DiskAnnContext *ctx) {
const std::vector<diskann_id_t> init_ids = get_init_ids(ctx);
int ret = iterate_to_fixed_point(init_ids, ctx);
if (ret != 0) {
return ret;
}
auto &pool = ctx->expanded_nodes();
pool.erase(std::remove_if(pool.begin(), pool.end(),
[id](const auto &node) { return node.id == id; }),
pool.end());
ret = prune_neighbors(id, pool, pruned_list, ctx);
if (ret != 0) {
return ret;
}
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,72 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <mutex>
#include <zvec/core/framework/index_framework.h>
#include <zvec/core/framework/index_meta.h>
#include "diskann_context.h"
namespace zvec {
namespace core {
class DiskAnnAlgorithm {
public:
typedef std::unique_ptr<DiskAnnAlgorithm> UPointer;
public:
DiskAnnAlgorithm(DiskAnnEntity &entity, uint32_t max_degree);
public:
int add_node(diskann_id_t id, DiskAnnContext *ctx);
int prune_node(diskann_id_t id, DiskAnnContext *ctx);
private:
int search_neighbor_and_prune(diskann_id_t id,
std::vector<diskann_id_t> &pruned_list,
DiskAnnContext *ctx);
int iterate_to_fixed_point(const std::vector<diskann_id_t> &init_ids,
DiskAnnContext *ctx);
int prune_neighbors(diskann_id_t id, std::vector<Neighbor> &pool,
std::vector<diskann_id_t> &pruned_list,
DiskAnnContext *ctx);
int inter_insert(diskann_id_t id, std::vector<diskann_id_t> &pruned_list,
DiskAnnContext *ctx);
int occlude_list(diskann_id_t id, std::vector<Neighbor> &pool,
std::vector<diskann_id_t> &result, DiskAnnContext *ctx);
std::vector<diskann_id_t> get_init_ids(DiskAnnContext *ctx);
private:
static constexpr uint32_t kLockCnt{1U << 16};
static constexpr uint32_t kLockMask{kLockCnt - 1U};
DiskAnnEntity &entity_;
uint32_t max_degree_{DiskAnnEntity::kDefaultMaxDegree};
uint32_t max_candidate_size_{DiskAnnEntity::kDefaultMaxOcclusionSize};
std::vector<std::mutex> lock_pool_{};
//! Acquire the striped lock guarding the given id (RAII).
[[nodiscard]] std::unique_lock<std::mutex> lock_for(diskann_id_t id) {
return std::unique_lock<std::mutex>(lock_pool_[id & kLockMask]);
}
float alpha_{DiskAnnEntity::kDefaultAlpha};
bool saturate_graph_{true};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,630 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_builder.h"
#include <cstring>
#include <iostream>
#include <thread>
#include <vector>
#include <ailego/math/euclidean_distance_matrix.h>
#include <ailego/math/normalizer.h>
#include <ailego/pattern/defer.h>
#include <zvec/core/framework/index_error.h>
#include <zvec/core/interface/index_factory.h>
#include "algorithm/cluster/vector_mean.h"
#include "diskann_context.h"
#include "diskann_params.h"
namespace zvec {
namespace core {
int DiskAnnBuilder::init(const IndexMeta &meta, const ailego::Params &params) {
LOG_INFO("Begin DiskAnnBuilder::init");
params.get(PARAM_DISKANN_BUILDER_MAX_DEGREE, &max_degree_);
params.get(PARAM_DISKANN_BUILDER_LIST_SIZE, &list_size_);
params.get(PARAM_DISKANN_BUILDER_THREAD_COUNT, &build_thread_count_);
if (build_thread_count_ == 0) {
build_thread_count_ = std::thread::hardware_concurrency();
}
if (build_thread_count_ > std::thread::hardware_concurrency()) {
LOG_WARN("Build thread count [%s] greater than cpu cores %u",
PARAM_DISKANN_BUILDER_THREAD_COUNT.c_str(),
std::thread::hardware_concurrency());
}
uint32_t max_pq_chunk_num{0};
if (params.get(PARAM_DISKANN_BUILDER_MAX_PQ_CHUNK_NUM, &max_pq_chunk_num)) {
if (max_pq_chunk_num > meta.dimension()) {
LOG_ERROR(
"PQ Chunk Num larger than dimension, PQ Chunk Num: %d, Dimension: %d",
max_pq_chunk_num, meta.dimension());
return IndexError_InvalidArgument;
}
max_pq_chunk_num_ = max_pq_chunk_num;
}
if (params.has(PARAM_DISKANN_BUILDER_MEMORY_LIMIT)) {
params.get(PARAM_DISKANN_BUILDER_MEMORY_LIMIT, &memory_limit_);
if (memory_limit_ <= 0) {
LOG_ERROR("Invalid memory limit: %lf", memory_limit_);
return IndexError_InvalidArgument;
}
memory_limit_set_ = true;
}
if (params.has(PARAM_DISKANN_BUILDER_MAX_TRAIN_SAMPLE_COUNT)) {
params.get(PARAM_DISKANN_BUILDER_MAX_TRAIN_SAMPLE_COUNT,
&max_train_sample_count_);
}
if (params.has(PARAM_DISKANN_BUILDER_TRAIN_SAMPLE_RATIO)) {
params.get(PARAM_DISKANN_BUILDER_TRAIN_SAMPLE_RATIO, &train_sample_ratio_);
}
raw_meta_ = meta;
build_meta_ = meta;
if (meta.metric_name() == "InnerProduct") {
build_meta_.set_metric("SquaredEuclidean", 0, ailego::Params());
} else if (meta.metric_name() == "Cosine") {
build_meta_.set_metric("SquaredEuclidean", 0, ailego::Params());
if (meta.data_type() == IndexMeta::DataType::DT_FP32) {
build_meta_.set_dimension(meta.dimension() - 1);
} else {
build_meta_.set_dimension(meta.dimension() - 2);
}
}
metric_ = IndexFactory::CreateMetric(build_meta_.metric_name());
if (!metric_) {
LOG_ERROR("CreateMetric failed, name: %s",
build_meta_.metric_name().c_str());
return IndexError_NoExist;
}
int ret = metric_->init(build_meta_, build_meta_.metric_params());
if (ret != 0) {
LOG_ERROR("IndexMeasure init failed, ret=%d", ret);
return ret;
}
raw_meta_.set_builder("DiskAnnBuilder", DiskAnnEntity::kRevision, params);
ret = entity_.init(meta, max_degree_, list_size_, memory_limit_,
build_thread_count_);
if (ret != 0) {
return ret;
}
algo_ =
DiskAnnAlgorithm::UPointer(new DiskAnnAlgorithm(entity_, max_degree_));
trainer_ =
DiskAnnPqTrainer::UPointer(new DiskAnnPqTrainer(max_train_sample_count_));
state_ = BUILD_STATE_INITED;
return 0;
}
int DiskAnnBuilder::cleanup(void) {
return 0;
}
int DiskAnnBuilder::calculate_entry_point() {
size_t dimension = build_meta_.dimension();
if (build_meta_.data_type() != IndexMeta::DataType::DT_FP32 &&
build_meta_.data_type() != IndexMeta::DataType::DT_FP16) {
LOG_ERROR("Data type not supported");
return IndexError_InvalidArgument;
}
std::vector<float> centroid_fp32;
std::vector<ailego::Float16> centroid_fp16;
switch (build_meta_.data_type()) {
case IndexMeta::DataType::DT_FP32: {
centroid_fp32.resize(dimension);
NumericalVectorMean<float> accumulator(dimension);
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
accumulator.plus(entity_.get_vector(id), dimension * sizeof(float));
}
accumulator.mean(centroid_fp32.data(), dimension * sizeof(float));
break;
}
case IndexMeta::DataType::DT_FP16: {
centroid_fp16.resize(dimension);
NumericalVectorMean<ailego::Float16> accumulator(dimension);
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
accumulator.plus(entity_.get_vector(id),
dimension * sizeof(ailego::Float16));
}
accumulator.mean(centroid_fp16.data(),
dimension * sizeof(ailego::Float16));
break;
}
default:
return IndexError_Unsupported;
}
// compute all to one distance
diskann_id_t medoid_id = kInvalidId;
float min_dist = std::numeric_limits<float>::max();
switch (build_meta_.data_type()) {
case IndexMeta::DataType::DT_FP32:
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
const float *data_ptr =
reinterpret_cast<const float *>(entity_.get_vector(id));
float dist = 0.0f;
ailego::SquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
centroid_fp32.data(), data_ptr, dimension, &dist);
if (dist < min_dist) {
min_dist = dist;
medoid_id = id;
}
}
break;
case IndexMeta::DataType::DT_FP16:
for (size_t id = 0; id < entity_.doc_cnt(); id++) {
const ailego::Float16 *data_ptr =
reinterpret_cast<const ailego::Float16 *>(entity_.get_vector(id));
float dist = 0.0f;
ailego::SquaredEuclideanDistanceMatrix<ailego::Float16, 1, 1>::Compute(
centroid_fp16.data(), data_ptr, dimension, &dist);
if (dist < min_dist) {
min_dist = dist;
medoid_id = id;
}
}
break;
default:
return IndexError_Unsupported;
}
(*entity_.mutable_medoid()) = medoid_id;
LOG_INFO("Medoid Calculation Done. ID: %zu", (size_t)medoid_id);
return 0;
}
int DiskAnnBuilder::calculate_pq_chunk_num() {
size_t doc_cnt = holder_->count();
if (doc_cnt == 0) {
LOG_ERROR("Invalid Input. Empty Vecs.");
return IndexError_InvalidLength;
}
if (memory_limit_set_) {
size_t memory_limit_bytes = get_memory_in_bytes(memory_limit_);
size_t pq_chunk_num = std::floor(memory_limit_bytes / doc_cnt);
if (pq_chunk_num <= 0) {
LOG_ERROR("Insufficient memory limit for vec, memory: %zu, vec num: %zu",
memory_limit_bytes, doc_cnt);
return IndexError_InvalidArgument;
}
}
pq_chunk_num_ =
pq_chunk_num_ < max_pq_chunk_num_ ? pq_chunk_num_ : max_pq_chunk_num_;
// A chunk num of 0 (public API default) or the internal sentinel means
// "auto": quantize into half the dimensions. Resolve it before the
// upper-bound check so the default never reaches the divide-by-chunk path.
if (pq_chunk_num_ == 0 || pq_chunk_num_ == kDefaultPqChunkNum) {
pq_chunk_num_ = build_meta_.dimension() / 2;
LOG_INFO(
"No Chunk Num input. Quantizing %u dimension data into %u dimension.",
build_meta_.dimension(), pq_chunk_num_);
}
if (pq_chunk_num_ > build_meta_.dimension()) {
LOG_ERROR("PQ Chunk Num is more than dimension, chunk num: %u, dim: %u",
pq_chunk_num_, build_meta_.dimension());
return IndexError_InvalidArgument;
}
LOG_INFO("Quantizing %u dimension data into %u bytes.",
build_meta_.dimension(), pq_chunk_num_);
return 0;
}
int DiskAnnBuilder::build_internal(IndexThreads::Pointer threads) {
auto task_group = threads->make_group();
if (!task_group) {
LOG_ERROR("Failed to create task group");
return IndexError_Runtime;
}
std::atomic<uint64_t> finished{0};
for (size_t i = 0; i < threads->count(); ++i) {
task_group->submit(ailego::Closure ::New(this, &DiskAnnBuilder::do_build, i,
threads->count(), &finished));
}
while (!task_group->is_finished()) {
std::unique_lock<std::mutex> lk(mutex_);
cond_.wait_until(lk, std::chrono::system_clock::now() +
std::chrono::seconds(check_interval_secs_));
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to build index while waiting finish");
return errcode_;
}
LOG_INFO("Built cnt %zu, finished percent %.3f%%", (size_t)finished.load(),
finished.load() * 100.0f / entity_.doc_cnt());
}
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to build index while waiting finish");
return errcode_;
}
task_group->wait_finish();
return 0;
}
int DiskAnnBuilder::prune_internal(IndexThreads::Pointer threads) {
auto task_group = threads->make_group();
if (!task_group) {
LOG_ERROR("Failed to create task group");
return IndexError_Runtime;
}
std::atomic<uint64_t> finished{0};
for (size_t i = 0; i < threads->count(); ++i) {
task_group->submit(ailego::Closure ::New(this, &DiskAnnBuilder::do_prune, i,
threads->count(), &finished));
}
while (!task_group->is_finished()) {
std::unique_lock<std::mutex> lk(mutex_);
cond_.wait_until(lk, std::chrono::system_clock::now() +
std::chrono::seconds(check_interval_secs_));
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to purne index while waiting finish");
return errcode_;
}
LOG_INFO("Prune cnt %zu, finished percent %.3f%%", (size_t)finished.load(),
finished.load() * 100.0f / entity_.doc_cnt());
}
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to prune index while waiting finish");
return errcode_;
}
task_group->wait_finish();
return 0;
}
int DiskAnnBuilder::train_quantized_data(IndexThreads::Pointer threads) {
LOG_INFO("Starting Train: Chunk Num: %u", pq_chunk_num_);
ailego::ElapsedTime timer;
int ret = trainer_->train_quantized_data(
threads, holder_, build_meta_, entity_.pq_full_pivot_data(),
entity_.pq_centroid(), entity_.pq_chunk_offsets(), pq_chunk_num_);
if (ret != 0) {
LOG_ERROR("Train Quantized Data Error, ret=%d", ret);
return ret;
}
size_t pq_time = timer.milli_seconds();
LOG_INFO("Train Quantized Data Done, time: %zu ms", pq_time);
(*entity_.mutable_pq_meta()).full_pivot_data_size =
entity_.pq_full_pivot_data().size();
(*entity_.mutable_pq_meta()).centroid_data_size =
entity_.pq_centroid().size();
(*entity_.mutable_pq_meta()).chunk_num = pq_chunk_num_;
return 0;
}
int DiskAnnBuilder::generate_quantized_data(IndexThreads::Pointer threads) {
LOG_INFO("Starting PQ Generate: Query Memory Limit: %lf, Chunk Num: %u",
memory_limit_, pq_chunk_num_);
ailego::ElapsedTime timer;
int ret = trainer_->generate_quantized_data(
threads, holder_, build_meta_, entity_.pq_centroid(),
entity_.block_compressed_data(), pq_chunk_num_);
if (ret != 0) {
LOG_ERROR("Generate Quantized Data Error, ret=%d", ret);
return ret;
}
size_t pq_time = timer.milli_seconds();
LOG_INFO("Generate Quantized Data Done, time: %zu ms", pq_time);
return 0;
}
void DiskAnnBuilder::do_build(uint64_t idx, size_t step_size,
std::atomic<uint64_t> *finished) {
AILEGO_DEFER([&]() {
std::lock_guard<std::mutex> latch(mutex_);
cond_.notify_one();
});
DiskAnnContext *ctx = new (std::nothrow) DiskAnnContext(
build_meta_, metric_,
std::shared_ptr<DiskAnnEntity>(&entity_, [](DiskAnnEntity *) {}));
if (ailego_unlikely(ctx == nullptr)) {
if (!error_.exchange(true)) {
LOG_ERROR("Failed to create context");
errcode_ = IndexError_NoMemory;
}
return;
}
DiskAnnContext::Pointer auto_ptr(ctx);
ctx->init(DiskAnnContext::kBuilderContext, max_degree_, pq_chunk_num_,
build_meta_.element_size());
ctx->set_list_size(list_size_);
for (uint64_t id = idx; id < entity_.doc_cnt(); id += step_size) {
ctx->reset_query(entity_.get_vector(id));
int ret = algo_->add_node(id, ctx);
if (ailego_unlikely(ret != 0)) {
if (!error_.exchange(true)) {
LOG_ERROR("DiskAnn graph add node failed");
errcode_ = ret;
}
return;
}
ctx->clear();
(*finished)++;
}
}
void DiskAnnBuilder::do_prune(uint64_t idx, size_t step_size,
std::atomic<uint64_t> *finished) {
AILEGO_DEFER([&]() {
std::lock_guard<std::mutex> latch(mutex_);
cond_.notify_one();
});
DiskAnnContext *ctx = new (std::nothrow) DiskAnnContext(
build_meta_, metric_,
std::shared_ptr<DiskAnnEntity>(&entity_, [](DiskAnnEntity *) {}));
if (ailego_unlikely(ctx == nullptr)) {
if (!error_.exchange(true)) {
LOG_ERROR("Failed to create context");
errcode_ = IndexError_NoMemory;
}
return;
}
DiskAnnContext::Pointer auto_ptr(ctx);
ctx->init(DiskAnnContext::kBuilderContext, max_degree_, pq_chunk_num_,
build_meta_.element_size());
ctx->set_list_size(list_size_);
for (uint64_t id = idx; id < entity_.doc_cnt(); id += step_size) {
ctx->reset_query(entity_.get_vector(id));
int ret = algo_->prune_node(id, ctx);
if (ailego_unlikely(ret != 0)) {
if (!error_.exchange(true)) {
LOG_ERROR("DiskAnn graph add node failed");
errcode_ = ret;
}
return;
}
ctx->clear();
(*finished)++;
}
}
int DiskAnnBuilder::train(const IndexTrainer::Pointer & /*trainer*/) {
if (state_ != BUILD_STATE_INITED) {
LOG_ERROR("Init the builder before DiskAnnBuilder::train");
return IndexError_NoReady;
}
LOG_INFO("Begin DiskAnnBuilder::train by trainer");
stats_.set_trained_count(0UL);
stats_.set_trained_costtime(0UL);
state_ = BUILD_STATE_TRAINED;
LOG_INFO("End DiskAnnBuilder::train by trainer");
return 0;
}
int DiskAnnBuilder::train(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) {
if (state_ != BUILD_STATE_INITED) {
LOG_ERROR("Init the builder before DiskAnnBuilder::train");
return IndexError_NoReady;
}
LOG_INFO("Begin DiskAnnBuilder::train");
auto start_time = ailego::Monotime::MilliSeconds();
holder_ = std::move(holder);
LOG_INFO("Start to calculate chunk num");
int ret = calculate_pq_chunk_num();
if (ailego_unlikely(ret != 0)) {
return ret;
}
if (!threads) {
threads =
std::make_shared<SingleQueueIndexThreads>(build_thread_count_, false);
if (!threads) {
return IndexError_NoMemory;
}
}
ret = train_quantized_data(threads);
if (ailego_unlikely(ret != 0)) {
return ret;
}
stats_.set_trained_count(holder_->count());
stats_.set_trained_costtime(ailego::Monotime::MilliSeconds() - start_time);
state_ = BUILD_STATE_TRAINED;
holder_.reset();
LOG_INFO("End DiskAnnBuilder::train");
return 0;
}
int DiskAnnBuilder::do_norm(const void *data_ptr, std::string *norm_data) {
size_t dimension = build_meta_.dimension();
const float *float_data_ptr = reinterpret_cast<const float *>(data_ptr);
norm_data->resize(dimension * sizeof(float));
float *output_buf = reinterpret_cast<float *>(&((*norm_data)[0]));
std::memcpy(output_buf, float_data_ptr, dimension * sizeof(float));
float norm = 0.0f;
ailego::Normalizer<float>::L2(output_buf, dimension, &norm);
return 0;
}
int DiskAnnBuilder::build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) {
LOG_INFO("Start DiskAnnBuilder::build");
auto start_time = ailego::Monotime::MilliSeconds();
holder_ = holder;
if (!threads) {
threads =
std::make_shared<SingleQueueIndexThreads>(build_thread_count_, false);
if (!threads) {
return IndexError_NoMemory;
}
}
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator for holder failed");
return IndexError_Runtime;
}
if (ailego_unlikely(holder->count() == 0)) {
LOG_ERROR("Holder is empty");
return IndexError_Runtime;
}
int ret = entity_.reserve_space(holder->count());
error_ = false;
while (iter->is_valid()) {
ret = entity_.add_vector(iter->key(), iter->data());
if (ailego_unlikely(ret != 0)) {
return ret;
}
iter->next();
}
LOG_INFO("Finished saving vector");
LOG_INFO("Start to calculate entrypoint");
ret = calculate_entry_point();
if (ailego_unlikely(ret != 0)) {
return ret;
}
LOG_INFO("Start to build vamana graph");
ret = build_internal(threads);
if (ret != 0) {
return ret;
}
LOG_INFO("Start final cleanup..");
ret = prune_internal(threads);
if (ret != 0) {
return ret;
}
LOG_INFO("Start to generate quantized data");
ret = generate_quantized_data(threads);
if (ailego_unlikely(ret != 0)) {
return ret;
}
state_ = BUILD_STATE_BUILT;
stats_.set_built_count(entity_.doc_cnt());
stats_.set_built_costtime(ailego::Monotime::MilliSeconds() - start_time);
LOG_INFO("End DiskAnnBuilder::build");
return 0;
}
int DiskAnnBuilder::dump(const IndexDumper::Pointer &dumper) {
if (state_ != BUILD_STATE_BUILT) {
LOG_INFO("Build the index before DiskAnnBuilder::dump");
return IndexError_NoReady;
}
LOG_INFO("Begin DiskAnnBuilder::dump");
raw_meta_.set_searcher("DiskAnnSearcher", 0, ailego::Params());
auto start_time = ailego::Monotime::MilliSeconds();
int ret = IndexHelper::SerializeToDumper(raw_meta_, dumper.get());
if (ret != 0) {
LOG_ERROR("Failed to serialize meta into dumper.");
return ret;
}
ret = entity_.dump(holder_, raw_meta_, dumper);
if (ret != 0) {
LOG_ERROR("Index dump failed, ret: %u", ret);
return IndexError_Runtime;
}
stats_.set_dumped_count(holder_->count());
stats_.set_dumped_costtime(ailego::Monotime::MilliSeconds() - start_time);
LOG_INFO("DiskAnnBuilder::dump");
return 0;
}
INDEX_FACTORY_REGISTER_BUILDER(DiskAnnBuilder);
} // namespace core
} // namespace zvec
@@ -0,0 +1,131 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/ailego/parallel/thread_pool.h>
#include <zvec/core/framework/index_builder.h>
#include "diskann_algorithm.h"
#include "diskann_builder_entity.h"
#include "diskann_pq_trainer.h"
namespace zvec {
namespace core {
class DiskAnnBuilder : public IndexBuilder {
public:
//! Constructor
DiskAnnBuilder() = default;
//! Initialize the builder
virtual int init(const IndexMeta &meta,
const ailego::Params &params) override;
//! Cleanup the builder
virtual int cleanup(void) override;
//! Train the data
virtual int train(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
//! Train the data
virtual int train(const IndexTrainer::Pointer &trainer) override;
//! Build the index
virtual int build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
//! Dump index into storage
virtual int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
return stats_;
}
int do_norm(const void *data_ptr, std::string *norm_data);
private:
int train_quantized_data(IndexThreads::Pointer threads);
int generate_quantized_data(IndexThreads::Pointer threads);
int build_internal(IndexThreads::Pointer threads);
int prune_internal(IndexThreads::Pointer threads);
void do_build(uint64_t idx, size_t step_size,
std::atomic<uint64_t> *finished);
void do_prune(uint64_t idx, size_t step_size,
std::atomic<uint64_t> *finished);
int calculate_entry_point();
int calculate_pq_chunk_num();
double get_memory_in_bytes(double search_ram_budget) {
return search_ram_budget * 1024 * 1024 * 1024;
}
private:
enum BUILD_STATE {
BUILD_STATE_INIT = 0,
BUILD_STATE_INITED = 1,
BUILD_STATE_TRAINED = 2,
BUILD_STATE_BUILT = 3
};
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
constexpr static uint32_t kDefaultListSize = 50U;
constexpr static uint32_t kDefaultMaxDegree = 100U;
constexpr static uint32_t kDefaultPqChunkNum = -1U;
std::string data_file_;
uint32_t max_degree_{kDefaultMaxDegree};
uint32_t list_size_{kDefaultListSize};
double memory_limit_{0.0};
bool memory_limit_set_{false};
uint32_t max_pq_chunk_num_{kDefaultPqChunkNum};
uint32_t pq_chunk_num_{kDefaultPqChunkNum};
uint32_t build_thread_count_{0};
uint32_t max_train_sample_count_{PQTable::kMaxTrainSampleCount};
double train_sample_ratio_{PQTable::kTrainSampleRatio};
std::string universal_label_{""};
std::string codebook_prefix_{""};
std::string index_path_prefix_{"./diskann"};
BUILD_STATE state_;
Stats stats_;
int errcode_{0};
std::atomic_bool error_{false};
IndexMetric::Pointer metric_{};
std::mutex mutex_{};
std::condition_variable cond_{};
IndexMeta raw_meta_;
IndexMeta build_meta_;
DiskAnnBuilderEntity entity_{};
IndexHolder::Pointer holder_;
DiskAnnAlgorithm::UPointer algo_;
DiskAnnPqTrainer::UPointer trainer_;
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,643 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_builder_entity.h"
#include <iostream>
#include "diskann_algorithm.h"
#include "diskann_util.h"
namespace zvec {
namespace core {
int DiskAnnBuilderEntity::init(const IndexMeta &meta, uint32_t max_degree,
uint32_t list_size, double memory_limit,
uint32_t build_threads) {
meta_ = meta;
max_degree_ = max_degree;
list_size_ = list_size;
memory_limit_ = memory_limit;
num_threads_ = build_threads;
max_build_degree_ = max_degree_ * kDefaultGraphSlackFactor;
neighbor_size_ = sizeof(uint32_t) + max_build_degree_ * sizeof(diskann_id_t);
return 0;
}
int DiskAnnBuilderEntity::reserve_space(uint32_t docs) {
vectors_buffer_.reserve(meta_.element_size() * docs);
keys_buffer_.reserve(sizeof(diskann_key_t) * docs);
neighbors_buffer_.reserve(neighbor_size_ * docs);
return 0;
}
int DiskAnnBuilderEntity::add_vector(diskann_key_t key, const void *vec) {
vectors_buffer_.append(reinterpret_cast<const char *>(vec),
meta_.element_size());
keys_buffer_.append(reinterpret_cast<const char *>(&key), sizeof(key));
uint32_t neighbor_cnt = 0;
std::vector<diskann_id_t> neighbor{max_build_degree_, 0};
neighbors_buffer_.append(reinterpret_cast<const char *>(&neighbor_cnt),
sizeof(uint32_t));
neighbors_buffer_.append(reinterpret_cast<const char *>(neighbor.data()),
sizeof(diskann_id_t) * max_build_degree_);
(*mutable_doc_cnt())++;
return 0;
}
const void *DiskAnnBuilderEntity::get_vector(diskann_id_t id) const {
size_t offset = (size_t)id * meta_.element_size();
return vectors_buffer_.data() + offset;
}
diskann_key_t DiskAnnBuilderEntity::get_key(diskann_id_t id) const {
size_t offset = (size_t)id * sizeof(diskann_key_t);
return *(
reinterpret_cast<const diskann_key_t *>(keys_buffer_.data() + offset));
}
//! Get vector local id by key
diskann_id_t DiskAnnBuilderEntity::get_id(diskann_key_t /*key*/) const {
LOG_ERROR("DiskAnnBuilderEntity::get_id not implemented.");
return kInvalidId;
}
std::pair<uint32_t, const diskann_id_t *> DiskAnnBuilderEntity::get_neighbors(
diskann_id_t id) const {
size_t offset = (size_t)id * neighbor_size_;
const uint8_t *start_ptr =
reinterpret_cast<const uint8_t *>(neighbors_buffer_.data()) + offset;
uint32_t neighbor_cnt = *(reinterpret_cast<const uint32_t *>(start_ptr));
const diskann_id_t *neighbors =
reinterpret_cast<const diskann_id_t *>(start_ptr + sizeof(uint32_t));
return std::make_pair(neighbor_cnt, neighbors);
}
int DiskAnnBuilderEntity::set_neighbors(
diskann_id_t id, const std::vector<diskann_id_t> &neighbor_ids) {
size_t offset = (size_t)id * neighbor_size_;
uint8_t *start_ptr =
reinterpret_cast<uint8_t *>(&neighbors_buffer_[0]) + offset;
uint32_t neighbor_cnt = neighbor_ids.size();
memcpy(start_ptr + sizeof(uint32_t), neighbor_ids.data(),
sizeof(diskann_id_t) * neighbor_cnt);
memcpy(start_ptr, &neighbor_cnt, sizeof(uint32_t));
if (max_observed_degree_ < neighbor_cnt) {
max_observed_degree_ = neighbor_cnt;
}
return 0;
}
int DiskAnnBuilderEntity::add_neighbor(diskann_id_t id,
diskann_id_t neighbor_id) {
size_t offset = (size_t)id * neighbor_size_;
uint8_t *start_ptr =
reinterpret_cast<uint8_t *>(&neighbors_buffer_[0]) + offset;
uint32_t neighbor_cnt = *reinterpret_cast<uint32_t *>(start_ptr);
memcpy(start_ptr + sizeof(uint32_t) + sizeof(diskann_id_t) * neighbor_cnt,
&neighbor_id, sizeof(diskann_id_t));
neighbor_cnt += 1;
memcpy(start_ptr, &neighbor_cnt, sizeof(uint32_t));
if (max_observed_degree_ < neighbor_cnt) {
max_observed_degree_ = neighbor_cnt;
}
return 0;
}
int64_t DiskAnnBuilderEntity::dump_segment(const IndexDumper::Pointer &dumper,
const std::string &segment_id,
const void *data,
size_t size) const {
size_t len = dumper->write(data, size);
if (len != size) {
LOG_ERROR("Dump segment %s data failed, expect: %lu, actual: %lu",
segment_id.c_str(), size, len);
return IndexError_WriteData;
}
size_t padding_size = AlignSize(size) - size;
if (padding_size > 0) {
std::string padding(padding_size, '\0');
if (dumper->write(padding.data(), padding_size) != padding_size) {
LOG_ERROR("Append padding failed, size %lu", padding_size);
return IndexError_WriteData;
}
}
uint32_t crc = ailego::Crc32c::Hash(data, size);
int ret = dumper->append(segment_id, size, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
return ret;
}
return 0;
}
int DiskAnnBuilderEntity::dump_pq_meta_segment(
const IndexDumper::Pointer &dumper) const {
uint32_t crc = 0U;
// write meta
size_t size_pq_meta = dumper->write(&pq_meta_, sizeof(DiskAnnPqMeta));
if (size_pq_meta != sizeof(DiskAnnPqMeta)) {
LOG_ERROR("Failed to dump PQ meta data, expect: %lu, actual: %lu",
sizeof(DiskAnnPqMeta), size_pq_meta);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(&pq_meta_, sizeof(DiskAnnPqMeta), crc);
// write full pivot data
size_t size_full_pivot_data =
dumper->write(pq_full_pivot_data_.data(), pq_meta_.full_pivot_data_size);
if (size_full_pivot_data != pq_meta_.full_pivot_data_size) {
LOG_ERROR("Failed to dump full pivot data, expect: %zu, actual: %zu",
(size_t)pq_meta_.full_pivot_data_size, size_full_pivot_data);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(pq_full_pivot_data_.data(),
pq_meta_.full_pivot_data_size, crc);
// write centroid num
size_t size_centroid =
dumper->write(pq_centroid_.data(), pq_meta_.centroid_data_size);
if (size_centroid != pq_meta_.centroid_data_size) {
LOG_ERROR("Failed to dump centroid num, expect: %zu, actual: %zu",
(size_t)pq_meta_.centroid_data_size, size_centroid);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(pq_centroid_.data(), pq_meta_.centroid_data_size,
crc);
// write chunk offset
size_t size_chunk_offset = dumper->write(
pq_chunk_offsets_.data(), (pq_meta_.chunk_num + 1) * sizeof(uint32_t));
if (size_chunk_offset != (pq_meta_.chunk_num + 1) * sizeof(uint32_t)) {
LOG_ERROR("Failed to dump centroid num, expect: %zu, actual: %zu",
(size_t)((pq_meta_.chunk_num + 1) * sizeof(uint32_t)),
size_chunk_offset);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(pq_chunk_offsets_.data(),
(pq_meta_.chunk_num + 1) * sizeof(uint32_t), crc);
// write size
size_t size_total =
size_pq_meta + size_full_pivot_data + size_centroid + size_chunk_offset;
// write pad
size_t padding_size = AlignSize(size_total) - size_total;
if (padding_size > 0) {
std::string padding(padding_size, '\0');
if (dumper->write(padding.data(), padding_size) != padding_size) {
LOG_ERROR("Append padding failed, size %lu", padding_size);
return IndexError_WriteData;
}
}
int ret =
dumper->append(kDiskAnnPqMetaSegmentId, size_total, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump PQ segment failed, ret %d", ret);
return ret;
}
return 0;
}
int DiskAnnBuilderEntity::dump_pq_data_segment(
const IndexDumper::Pointer &dumper) const {
uint64_t doc_cnt = meta_header_.doc_cnt;
uint64_t chunk_num = pq_meta_.chunk_num;
uint32_t crc = 0U;
// write pq data
size_t size_total =
dumper->write(block_compressed_data_.data(), doc_cnt * chunk_num);
if (size_total != doc_cnt * chunk_num) {
LOG_ERROR("Failed to dump block compressed data, expect: %zu, actual: %zu",
(size_t)(doc_cnt * chunk_num), size_total);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(block_compressed_data_.data(), doc_cnt * chunk_num,
crc);
// write pad
size_t padding_size = AlignSize(size_total) - size_total;
if (padding_size > 0) {
std::string padding(padding_size, '\0');
if (dumper->write(padding.data(), padding_size) != padding_size) {
LOG_ERROR("Append padding failed, size %lu", padding_size);
return IndexError_WriteData;
}
}
int ret =
dumper->append(kDiskAnnPqDataSegmentId, size_total, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump PQ data segment failed, ret %d", ret);
return ret;
}
return 0;
}
int DiskAnnBuilderEntity::dump_dummy_segment(
const IndexDumper::Pointer &dumper) const {
// to make offset aligned with 4K
size_t dumper_header_size = dumper->size();
size_t dummy_size =
DiskAnnUtil::round_up(dumper_header_size, DiskAnnUtil::kSectorSize) -
dumper_header_size;
if (dummy_size != 0) {
std::string dummpy_data(dummy_size, '\0');
if (dumper->write(dummpy_data.data(), dummy_size) != dummy_size) {
LOG_ERROR("write dummy failed, size %lu", dummy_size);
return IndexError_WriteData;
}
int ret = dumper->append(kDiskAnnDummpySegmentId, dummy_size, 0, 0);
if (ret != 0) {
LOG_ERROR("Dump dummy data segment failed, ret %d", ret);
return ret;
}
}
return 0;
}
int DiskAnnBuilderEntity::dump_key_segment(
const IndexDumper::Pointer &dumper) const {
//! Dump keys
size_t key_segment_size = doc_cnt() * sizeof(diskann_key_t);
int64_t keys_size = dump_segment(dumper, kDiskAnnKeySegmentId,
keys_buffer_.data(), key_segment_size);
if (keys_size < 0) {
return keys_size;
}
return 0;
}
int DiskAnnBuilderEntity::dump_key_mapping_segment(
const IndexDumper::Pointer &dumper) const {
std::vector<diskann_id_t> mapping(doc_cnt());
const diskann_key_t *keys = reinterpret_cast<diskann_key_t *>(
const_cast<char *>(keys_buffer_.data()));
std::iota(mapping.begin(), mapping.end(), 0U);
std::sort(mapping.begin(), mapping.end(),
[&](diskann_id_t i, diskann_id_t j) { return keys[i] < keys[j]; });
size_t size = mapping.size() * sizeof(diskann_id_t);
int64_t ret =
dump_segment(dumper, kDiskAnnKeyMappingSegmentId, mapping.data(), size);
if (ret != 0) {
LOG_ERROR("Dump vectors segment failed");
return ret;
}
return 0;
}
int DiskAnnBuilderEntity::dump_entrypoint_segment(
const IndexDumper::Pointer &dumper) const {
std::string entrypoint_buffer;
size_t size = sizeof(uint32_t) + entrypoints_.size() * sizeof(diskann_id_t);
entrypoint_buffer.resize(size);
uint8_t *buffer_ptr = reinterpret_cast<uint8_t *>(&entrypoint_buffer[0]);
uint32_t point_cnt = entrypoints_.size();
memcpy(buffer_ptr, &point_cnt, sizeof(uint32_t));
memcpy(buffer_ptr + sizeof(uint32_t), entrypoints_.data(),
entrypoints_.size() * sizeof(diskann_id_t));
int64_t ret = dump_segment(dumper, kDiskAnnEntryPointSegmentId,
entrypoint_buffer.data(), size);
if (ret != 0) {
LOG_ERROR("Dump entrypoint segment failed");
return ret;
}
return 0;
}
int DiskAnnBuilderEntity::dump(IndexHolder::Pointer holder, IndexMeta &meta,
const IndexDumper::Pointer &dumper) {
uint64_t doc_cnt = holder->count();
uint64_t max_node_size =
(uint64_t)max_observed_degree_ * sizeof(diskann_id_t) + sizeof(uint32_t) +
meta_.element_size();
uint64_t node_per_sector =
DiskAnnUtil::kSectorSize /
max_node_size; // 0 if max_node_size > DiskAnnUtil::kSectorSize
std::string node_buf;
node_buf.resize(max_node_size);
diskann_id_t *neighbor_buf =
(diskann_id_t *)(node_buf.data() + (meta_.element_size()) +
sizeof(uint32_t));
LOG_INFO(
"Dump Data, medoid: %zu, max node size: %zu, node per sector: %zu, "
"max observed degree: %zu",
(size_t)medoid(), (size_t)max_node_size, (size_t)node_per_sector,
(size_t)max_observed_degree_);
// write a dummy segment to make data align
int ret = dump_dummy_segment(dumper);
if (ret != 0) {
LOG_ERROR("Dump dummy segment failed");
return ret;
}
// dump data by sector
size_t write_size = 0;
uint32_t crc = 0U;
size_t len = 0;
// no need to write first sector
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator for holder failed");
return IndexError_Runtime;
}
uint64_t index_size = 0;
uint32_t neighbor_num;
if (node_per_sector > 0) {
uint64_t sector_num =
DiskAnnUtil::round_up(doc_cnt, node_per_sector) / node_per_sector;
diskann_id_t cur_node_id = 0;
std::string sector_buf;
sector_buf.resize(DiskAnnUtil::kSectorSize);
for (uint64_t sector = 0; sector < sector_num; sector++) {
if (sector != 0 && sector % 100000 == 0) {
LOG_INFO("Sector #%zu written", (size_t)sector);
}
memset(&(sector_buf[0]), 0, DiskAnnUtil::kSectorSize);
for (uint64_t sector_node_id = 0;
sector_node_id < node_per_sector && cur_node_id < doc_cnt;
sector_node_id++) {
memset(&(node_buf[0]), 0, max_node_size);
auto neighbors = get_neighbors(cur_node_id);
neighbor_num = neighbors.first;
ailego_assert(neighbor_num > 0);
ailego_assert(neighbor_num <= max_observed_degree_);
memcpy(&(neighbor_buf[0]), neighbors.second,
neighbors.first * sizeof(diskann_id_t));
if (iter->is_valid()) {
const void *vec = iter->data();
memcpy(&(node_buf[0]), vec, meta.element_size());
iter->next();
} else {
return IndexError_Runtime;
}
// write neighbor num
*(uint32_t *)(node_buf.data() + meta_.element_size()) = neighbor_num;
// write neighbor buffer
memcpy(&(node_buf[0]) + meta_.element_size() + sizeof(uint32_t),
neighbor_buf, neighbor_num * sizeof(diskann_id_t));
// get offset into sector_buf
char *sector_node_buf = &sector_buf[sector_node_id * max_node_size];
// copy node buf into sector_node_buf
memcpy(sector_node_buf, node_buf.data(), max_node_size);
cur_node_id++;
}
// flush sector to disk
len = dumper->write(sector_buf.data(), DiskAnnUtil::kSectorSize);
if (len != DiskAnnUtil::kSectorSize) {
LOG_ERROR("Write Vector Error, write=%zu, expect=%zu", len,
(size_t)DiskAnnUtil::kSectorSize);
return IndexError_WriteData;
}
write_size += DiskAnnUtil::kSectorSize;
crc = ailego::Crc32c::Hash(sector_buf.data(), DiskAnnUtil::kSectorSize,
crc);
}
LOG_INFO("Total Sector #%zu written", (size_t)sector_num);
index_size = sector_num * DiskAnnUtil::kSectorSize;
} else {
// Write multi-sector nodes
std::string multisector_buf;
multisector_buf.resize(
DiskAnnUtil::round_up(max_node_size, DiskAnnUtil::kSectorSize));
uint64_t sector_num_per_node =
DiskAnnUtil::div_round_up(max_node_size, DiskAnnUtil::kSectorSize);
for (uint64_t i = 0; i < doc_cnt; i++) {
if (i != 0 && (i * sector_num_per_node) % 100000 == 0) {
LOG_INFO("Sector # %zu written", (size_t)(i * sector_num_per_node));
}
memset(&(multisector_buf[0]), 0,
sector_num_per_node * DiskAnnUtil::kSectorSize);
memset(&(node_buf[0]), 0, max_node_size);
auto neighbors = get_neighbors(i);
neighbor_num = neighbors.first;
ailego_assert(neighbor_num > 0);
ailego_assert(neighbor_num <= max_observed_degree_);
// read node's nhood
memcpy((char *)neighbor_buf, neighbors.second,
neighbor_num * sizeof(diskann_id_t));
if (iter->is_valid()) {
const void *vec = iter->data();
memcpy(&(multisector_buf[0]), vec, meta.element_size());
iter->next();
} else {
return IndexError_Runtime;
}
// write neighbor
*(uint32_t *)(&(multisector_buf[0]) + meta_.element_size()) =
neighbor_num;
// write nhood next
memcpy(&(multisector_buf[0]) + meta_.element_size() + sizeof(uint32_t),
neighbor_buf, neighbor_num * sizeof(diskann_id_t));
// flush sector to disk
len = dumper->write(multisector_buf.data(),
sector_num_per_node * DiskAnnUtil::kSectorSize);
if (len != sector_num_per_node * DiskAnnUtil::kSectorSize) {
LOG_ERROR("Write Vector Error, write=%zu, expect=%zu", len,
(size_t)(sector_num_per_node * DiskAnnUtil::kSectorSize));
return IndexError_WriteData;
}
write_size += sector_num_per_node * DiskAnnUtil::kSectorSize;
crc = ailego::Crc32c::Hash(multisector_buf.data(),
sector_num_per_node * DiskAnnUtil::kSectorSize,
crc);
}
LOG_INFO("Total Sector #%zu written",
(size_t)(doc_cnt * sector_num_per_node));
index_size = doc_cnt * sector_num_per_node * DiskAnnUtil::kSectorSize;
}
size_t padding_size = AlignSize(write_size) - write_size;
if (padding_size > 0) {
std::string padding(padding_size, '\0');
if (dumper->write(padding.data(), padding_size) != padding_size) {
LOG_ERROR("Append padding failed, size %lu", padding_size);
return IndexError_WriteData;
}
}
ret = dumper->append(kDiskAnnVectorSegmentId, write_size, 0UL, crc);
if (ret != 0) {
LOG_ERROR("Dump vectors segment failed, ret %d", ret);
return ret;
}
// dump diskann meta
meta_header_.doc_cnt = doc_cnt;
meta_header_.ndims = meta_.dimension();
meta_header_.medoid = medoid();
meta_header_.max_node_size = max_node_size;
meta_header_.max_degree = max_observed_degree_;
meta_header_.node_per_sector = node_per_sector;
meta_header_.vamana_frozen_num = 0;
meta_header_.vamana_frozen_loc = medoid();
meta_header_.append_reorder_data = 0;
meta_header_.index_size = index_size;
ret = dump_segment(dumper, kDiskAnnMetaSegmentId, &meta_header_,
sizeof(DiskAnnMetaHeader));
if (ret != 0) {
LOG_ERROR("Dump vectors segment failed");
return ret;
}
// dump pq meta
ret = dump_pq_meta_segment(dumper);
if (ret != 0) {
LOG_ERROR("Dump pq meta segment failed");
return ret;
}
// dump pq data
ret = dump_pq_data_segment(dumper);
if (ret != 0) {
LOG_ERROR("Dump pq data segment failed");
return ret;
}
// dump key
ret = dump_key_segment(dumper);
if (ret != 0) {
LOG_ERROR("Dump key segment failed");
return ret;
}
// dump key mapping
ret = dump_key_mapping_segment(dumper);
if (ret != 0) {
LOG_ERROR("Dump key mapping segment failed");
return ret;
}
// dump entrypint
ret = dump_entrypoint_segment(dumper);
if (ret != 0) {
LOG_ERROR("Dump entrypoint segment failed");
return ret;
}
LOG_INFO("DiskAnn Index File Dumped");
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,107 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/ailego/parallel/thread_pool.h>
#include <zvec/core/framework/index_holder.h>
#include "diskann_entity.h"
namespace zvec {
namespace core {
// wrapper class aligned with diskann
class DiskAnnBuilderEntity : public DiskAnnEntity {
public:
using Pointer = std::shared_ptr<DiskAnnBuilderEntity>;
DiskAnnBuilderEntity() = default;
virtual ~DiskAnnBuilderEntity() = default;
public:
int add_vector(diskann_key_t key, const void *vec) override;
std::pair<uint32_t, const diskann_id_t *> get_neighbors(
diskann_id_t id) const override;
int set_neighbors(diskann_id_t id,
const std::vector<diskann_id_t> &neighbor_ids) override;
int add_neighbor(diskann_id_t id, diskann_id_t neighbor_id) override;
diskann_id_t get_id(diskann_key_t key) const override;
diskann_key_t get_key(diskann_id_t id) const override;
const void *get_vector(diskann_id_t id) const override;
public:
int init(const IndexMeta &meta, uint32_t max_degree, uint32_t list_size,
double memory_limit, uint32_t build_threads);
int dump(IndexHolder::Pointer holder, IndexMeta &meta,
const IndexDumper::Pointer &dumper);
int64_t dump_segment(const IndexDumper::Pointer &dumper,
const std::string &segment_id, const void *data,
size_t size) const;
int dump_dummy_segment(const IndexDumper::Pointer &dumper) const;
int dump_pq_meta_segment(const IndexDumper::Pointer &dumper) const;
int dump_pq_data_segment(const IndexDumper::Pointer &dumper) const;
int dump_key_mapping_segment(const IndexDumper::Pointer &dumper) const;
int dump_entrypoint_segment(const IndexDumper::Pointer &dumper) const;
int dump_key_segment(const IndexDumper::Pointer &dumper) const;
int reserve_space(uint32_t docs);
std::vector<uint8_t> &pq_full_pivot_data() {
return pq_full_pivot_data_;
}
std::vector<uint8_t> &pq_centroid() {
return pq_centroid_;
}
std::vector<uint32_t> &pq_chunk_offsets() {
return pq_chunk_offsets_;
}
std::vector<uint8_t> &block_compressed_data() {
return block_compressed_data_;
}
private:
uint32_t max_degree_{0};
uint32_t list_size_{0};
double memory_limit_{0};
uint32_t num_threads_{0};
uint32_t max_build_degree_{0};
uint32_t max_observed_degree_{0};
uint32_t neighbor_size_{0};
std::string mem_index_file_{""};
std::string index_path_prefix_{""};
std::string vectors_buffer_{};
std::string keys_buffer_{};
std::string neighbors_buffer_{};
std::vector<diskann_id_t> entrypoints_{};
IndexMeta meta_;
std::vector<uint8_t> pq_full_pivot_data_;
std::vector<uint8_t> pq_centroid_;
std::vector<uint32_t> pq_chunk_offsets_;
std::vector<uint8_t> block_compressed_data_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,134 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_context.h"
#include <chrono>
#include "diskann_pq_table.h"
#include "diskann_util.h"
namespace zvec {
namespace core {
DiskAnnContext::DiskAnnContext(const IndexMeta &meta,
const IndexMetric::Pointer &measure,
const DiskAnnEntity::Pointer &entity)
: dc_(entity.get(), measure, meta.dimension()), entity_{entity} {}
int DiskAnnContext::init(ContextType type, uint32_t graph_degree,
uint32_t pq_chunk_num, uint32_t element_size) {
type_ = type;
element_size_ = element_size;
pq_chunk_num_ = pq_chunk_num;
DiskAnnUtil::alloc_aligned((void **)&query_, element_size_, 32);
DiskAnnUtil::alloc_aligned((void **)&query_rotated_, element_size_, 32);
int ret;
switch (type) {
case kBuilderContext:
ret = visit_filter_.init(VisitFilter::ByteMap, entity_->doc_cnt(),
entity_->doc_cnt(), negative_probility_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
break;
case kSearcherContext:
ret = visit_filter_.init(filter_mode_, entity_->doc_cnt(),
entity_->doc_cnt(), negative_probility_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
DiskAnnUtil::alloc_aligned(
(void **)&pq_table_dist_buffer_,
PQTable::kPQCentroidNum * pq_chunk_num_ * sizeof(float), 256);
DiskAnnUtil::alloc_aligned((void **)&pq_coord_buffer_,
graph_degree * pq_chunk_num_ * sizeof(uint8_t),
256);
DiskAnnUtil::alloc_aligned((void **)&coord_buffer_, element_size_, 256);
DiskAnnUtil::alloc_aligned(
(void **)&sector_buffer_,
DiskAnnUtil::kMaxSectorReadNum * DiskAnnUtil::kSectorSize,
DiskAnnUtil::kSectorSize);
ret = setup_io_ctx(io_ctx_);
if (ret != 0) {
LOG_ERROR("setup io ctx error, ret=%d", ret);
return ret;
}
break;
default:
LOG_ERROR("Init context failed");
return IndexError_Runtime;
}
return 0;
}
DiskAnnContext::~DiskAnnContext() {
free(query_);
free(query_rotated_);
free(pq_table_dist_buffer_);
free(pq_coord_buffer_);
free(coord_buffer_);
free(sector_buffer_);
if (type_ == kSearcherContext) {
destroy_io_ctx(io_ctx_);
}
}
int DiskAnnContext::update_context(ContextType type, const IndexMeta &meta,
const IndexMetric::Pointer &measure,
const DiskAnnEntity::Pointer &entity,
uint32_t magic_num) {
if (ailego_unlikely(type != type_)) {
LOG_ERROR(
"DiskAnnContext does not support shared by different type, "
"src=%u dst=%u",
type_, type);
return IndexError_Unsupported;
}
magic_ = kInvalidMgic;
switch (type) {
case kBuilderContext:
LOG_ERROR("BuildContext does not support update");
return IndexError_NotImplemented;
case kSearcherContext:
break;
case kReducerContext:
break;
default:
LOG_ERROR("update context failed");
return IndexError_Runtime;
}
entity_ = entity;
dc_.update(measure, meta.dimension());
magic_ = magic_num;
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,372 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/core/framework/index_context.h>
#include "utility/topk_result_builder.h"
#include "diskann_dist_calculator.h"
#include "diskann_entity.h"
#include "diskann_file_reader.h"
#include "diskann_visit_filter.h"
namespace zvec {
namespace core {
struct SearchStats {
public:
float total_us = 0;
float io_us = 0;
float cpu_us = 0;
uint64_t disk_page_reads = 0;
uint64_t io_num = 0;
uint64_t dist_num = 0;
uint64_t cache_hits = 0;
uint64_t hop_num = 0;
};
class DiskAnnContext : public IndexContext,
public TopkResultBuilder<DiskAnnContext, TopkHeap> {
friend class TopkResultBuilder<DiskAnnContext, TopkHeap>;
public:
//! Index Context Pointer
using Pointer = std::unique_ptr<DiskAnnContext>;
enum ContextType {
kUnknownContext = 0,
kSearcherContext = 1,
kBuilderContext = 2,
kReducerContext = 3
};
//! Construct
DiskAnnContext(const IndexMeta &meta, const IndexMetric::Pointer &measure,
const DiskAnnEntity::Pointer &entity);
//! Destructor
virtual ~DiskAnnContext();
public:
//! Init
int init(ContextType type, uint32_t graph_degree, uint32_t pq_chunk_num,
uint32_t element_size);
//! Update context, the context may be shared by different searcher/streamer
int update_context(ContextType type, const IndexMeta &meta,
const IndexMetric::Pointer &measure,
const DiskAnnEntity::Pointer &entity, uint32_t magic_num);
//! Retrieve search result
virtual const IndexDocumentList &result(void) const override {
return results_[0];
}
//! Retrieve search result
virtual const IndexDocumentList &result(size_t idx) const override {
return results_[idx];
}
//! Retrieve result object for output
virtual IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(void) const override {
return group_results_[0];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(
size_t idx) const override {
return group_results_[idx];
}
virtual uint32_t magic(void) const override {
return magic_;
}
void set_magic(uint32_t magic) {
magic_ = magic;
}
//! Set mode of debug
virtual void set_debug_mode(bool enable) override {
debug_mode_ = enable;
}
//! Retrieve mode of debug
virtual bool debug_mode(void) const override {
return debug_mode_;
}
//! Retrieve string of debug
virtual std::string debug_string(void) const override {
return std::string("");
}
//! Update the parameters of context
virtual int update(const ailego::Params & /*params*/) override {
return 0;
}
inline DistCalculator &dist_calculator() {
return dc_;
}
public:
//! Set topk of search result
void set_topk(uint32_t val) override {
topk_ = val;
topk_heap_.limit(val);
}
void set_list_size(uint32_t list_size) {
list_size_ = list_size;
}
void set_fetch_vector(bool v) override {
fetch_vector_ = v;
}
//! Get topk
inline uint32_t topk() const override {
return topk_;
}
inline uint32_t list_size() const {
return list_size_;
}
inline void reset_query(const void *query) {
memcpy(query_, query, element_size_);
memcpy(query_rotated_, query, element_size_);
dc_.reset_query(query);
}
inline TopkHeap &topk_heap() {
return topk_heap_;
}
inline void *query() {
return query_;
}
inline void *query_rotated() {
return query_rotated_;
}
inline float *pq_table_dist_buffer() {
return pq_table_dist_buffer_;
}
inline void *pq_coord_buffer() {
return pq_coord_buffer_;
}
inline void *coord_buffer() {
return coord_buffer_;
}
inline void *sector_buffer() {
return sector_buffer_;
}
inline IOContext &io_ctx() {
return io_ctx_;
}
inline void resize_results(size_t size) {
if (group_by_search()) {
group_results_.resize(size);
} else {
results_.resize(size);
}
}
inline bool error() const {
return has_error_;
}
inline void set_error(bool err) {
has_error_ = err;
}
inline void clear() {
for (auto &it : results_) {
it.clear();
}
best_list_nodes_.clear();
expanded_nodes_.clear();
visit_filter_.clear();
has_error_ = false;
}
SearchStats &query_stats() {
return query_stats_;
}
const DiskAnnEntity &get_entity() const {
return *entity_;
}
NeighborPriorityQueue &best_list_nodes() {
return best_list_nodes_;
}
std::vector<Neighbor> &expanded_nodes() {
return expanded_nodes_;
}
std::vector<float> &occlude_factor() {
return occlude_factor_;
}
VisitFilter &visit_filter() {
return visit_filter_;
}
//! Reset context
void reset(void) override {
set_filter(nullptr);
reset_threshold();
set_fetch_vector(false);
set_group_params(0, 0);
reset_group_by();
}
inline std::map<std::string, TopkHeap> &group_topk_heaps() {
return group_topk_heaps_;
}
//! Get group topk
inline uint32_t group_topk() const {
return group_topk_;
}
//! Get group num
inline uint32_t group_num() const {
return group_num_;
}
//! Get if group by search
inline bool group_by_search() {
return group_num_ > 0;
}
//! Set group params
void set_group_params(uint32_t group_num, uint32_t group_topk) override {
group_num_ = group_num;
group_topk_ = group_topk;
topk_ = group_topk_ * group_num_;
topk_heap_.limit(topk_);
group_topk_heaps_.clear();
}
// topk_to_result / topk_to_single_result / topk_to_group_result are provided
// by TopkResultBuilder via the hooks below (result_score,
// emplace_result_doc).
using TopkResultBuilder<DiskAnnContext, TopkHeap>::topk_to_result;
void set_to_result(uint32_t idx, const std::vector<diskann_id_t> &result_ids,
const std::vector<float> &result_dists) {
if (result_ids.size() != result_dists.size()) {
return;
}
uint32_t size = result_ids.size();
for (uint32_t i = 0; i < size; ++i) {
results_[idx].emplace_back(result_ids[i], result_dists[i], 0);
}
}
private:
//! Hooks required by TopkResultBuilder.
std::vector<IndexDocumentList> &mutable_results() {
return results_;
}
std::vector<IndexGroupDocumentList> &mutable_group_results() {
return group_results_;
}
float result_score(const VectorInfo &info) const {
return info.dist_;
}
void emplace_result_doc(IndexDocumentList &docs, diskann_id_t id, float score,
const VectorInfo &info) {
if (fetch_vector_) {
docs.emplace_back(entity_->get_key(id), score, id, info.vec_);
} else {
docs.emplace_back(entity_->get_key(id), score, id);
}
}
private:
constexpr static uint32_t kInvalidMgic = -1U;
uint32_t type_{kUnknownContext};
DistCalculator dc_;
DiskAnnEntity::Pointer entity_;
uint32_t topk_{0};
uint32_t magic_{0U};
bool debug_mode_{false};
uint32_t pq_chunk_num_{0};
uint32_t element_size_{0};
uint32_t element_rotated_size_{0};
uint32_t list_size_{0};
TopkHeap topk_heap_{};
uint32_t group_topk_{0};
uint32_t group_num_{0};
std::map<std::string, TopkHeap> group_topk_heaps_{};
IOContext io_ctx_{0};
SearchStats query_stats_;
float *pq_table_dist_buffer_{nullptr};
void *pq_coord_buffer_{nullptr};
void *query_{nullptr};
void *query_rotated_{nullptr};
void *coord_buffer_{nullptr};
void *sector_buffer_{nullptr};
std::vector<IndexDocumentList> results_{};
std::vector<IndexGroupDocumentList> group_results_{};
bool fetch_vector_{false};
bool has_error_{false};
NeighborPriorityQueue best_list_nodes_;
std::vector<Neighbor> expanded_nodes_;
std::vector<float> occlude_factor_;
VisitFilter visit_filter_{};
uint32_t filter_mode_{VisitFilter::ByteMap};
float negative_probility_{DiskAnnEntity::kDefaultBFNegativeProbility};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,150 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <zvec/core/framework/index_context.h>
#include <zvec/core/framework/index_meta.h>
#include "diskann_entity.h"
namespace zvec {
namespace core {
class DistCalculator {
public:
typedef std::shared_ptr<DistCalculator> Pointer;
public:
//! Constructor
DistCalculator(const DiskAnnEntity *entity,
const IndexMetric::Pointer &measure, uint32_t dim)
: entity_(entity),
distance_(measure->distance()),
query_(nullptr),
dim_(dim),
compare_cnt_(0) {}
void update(const IndexMetric::Pointer &measure, uint32_t dim) {
distance_ = measure->distance();
dim_ = dim;
}
inline void update_distance(const IndexMetric::MatrixDistance &distance) {
distance_ = distance;
}
//! Reset query vector data
inline void reset_query(const void *query) {
error_ = false;
query_ = query;
}
//! Returns distance
inline dist_t dist(const void *vec_lhs, const void *vec_rhs) {
if (ailego_unlikely(vec_lhs == nullptr || vec_rhs == nullptr)) {
LOG_ERROR("Nullptr of dense vector");
error_ = true;
return 0.0f;
}
float score{0.0f};
distance_(vec_lhs, vec_rhs, dim_, &score);
return score;
}
//! Returns distance between query and vec.
inline dist_t dist(const void *vec) {
compare_cnt_++;
return dist(vec, query_);
}
inline dist_t dist(diskann_id_t id) {
compare_cnt_++;
const void *vec = entity_->get_vector(id);
if (ailego_unlikely(vec == nullptr)) {
LOG_ERROR("Get nullptr vector, id=%u", id);
error_ = true;
return 0.0f;
}
return dist(vec, query_);
}
inline dist_t dist(diskann_id_t lhs, diskann_id_t rhs) {
compare_cnt_++;
const void *vec_lhs = entity_->get_vector(lhs);
if (ailego_unlikely(vec_lhs == nullptr)) {
LOG_ERROR("Get nullptr vector, lhs id=%u", lhs);
error_ = true;
return 0.0f;
}
const void *vec_rhs = entity_->get_vector(rhs);
if (ailego_unlikely(vec_rhs == nullptr)) {
LOG_ERROR("Get nullptr vector, rhs id=%u", rhs);
error_ = true;
return 0.0f;
}
return dist(vec_lhs, vec_rhs);
}
dist_t operator()(const void *vec) {
return dist(vec);
}
inline void clear() {
compare_cnt_ = 0;
error_ = false;
}
inline void clear_compare_cnt() {
compare_cnt_ = 0;
}
inline bool error() const {
return error_;
}
//! Get distances compute times
inline uint32_t compare_cnt() const {
return compare_cnt_;
}
inline uint32_t dimension() const {
return dim_;
}
private:
DistCalculator(const DistCalculator &) = delete;
DistCalculator &operator=(const DistCalculator &) = delete;
private:
const DiskAnnEntity *entity_;
IndexMetric::MatrixDistance distance_;
const void *query_;
uint32_t dim_;
uint32_t compare_cnt_; // record distance compute times
bool error_{false};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,32 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_entity.h"
namespace zvec {
namespace core {
const std::string DiskAnnEntity::kDiskAnnVectorSegmentId = "diskann.vector";
const std::string DiskAnnEntity::kDiskAnnMetaSegmentId = "diskann.meta";
const std::string DiskAnnEntity::kDiskAnnPqMetaSegmentId = "diskann.pq_meta";
const std::string DiskAnnEntity::kDiskAnnPqDataSegmentId = "diskann.pq_data";
const std::string DiskAnnEntity::kDiskAnnDummpySegmentId = "diskann.dummy";
const std::string DiskAnnEntity::kDiskAnnKeyMappingSegmentId =
"diskann.key_mapping";
const std::string DiskAnnEntity::kDiskAnnEntryPointSegmentId =
"diskann.entrypoint";
const std::string DiskAnnEntity::kDiskAnnKeySegmentId = "diskann.key";
} // namespace core
} // namespace zvec
+244
View File
@@ -0,0 +1,244 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <utility>
#include <zvec/core/framework/index_framework.h>
namespace zvec {
namespace core {
using dist_t = float;
using diskann_key_t = uint64_t;
using diskann_id_t = uint32_t;
constexpr diskann_id_t kInvalidId = static_cast<diskann_id_t>(-1);
constexpr diskann_key_t kInvalidKey = static_cast<key_t>(-1);
struct VectorInfo {
float dist_;
std::string vec_;
VectorInfo() = default;
VectorInfo(float dist, const std::string &vec) : dist_{dist}, vec_{vec} {}
VectorInfo(float dist, std::string &&vec)
: dist_{dist}, vec_{std::move(vec)} {}
};
/*! Key Value Vecotr Heap Comparer
*/
struct KeyValueVectorHeapComparer {
//! Function call
bool operator()(const std::pair<diskann_id_t, VectorInfo> &lhs,
const std::pair<diskann_id_t, VectorInfo> &rhs) const {
return compare_(lhs.second.dist_, rhs.second.dist_);
}
private:
std::less<float> compare_;
};
/*! Key Value Vector Heap
*/
using TopkHeap = ailego::Heap<std::pair<diskann_id_t, VectorInfo>,
KeyValueVectorHeapComparer>;
struct DiskAnnMetaHeader {
public:
uint64_t doc_cnt;
uint64_t ndims;
uint64_t medoid;
uint64_t max_node_size;
uint64_t max_degree;
uint64_t node_per_sector;
uint64_t vamana_frozen_num;
uint64_t vamana_frozen_loc;
uint64_t append_reorder_data;
uint64_t index_size;
uint8_t reserved[4016]; /// pad DiskAnnMetaHeader to 4096 bytes
DiskAnnMetaHeader() {
clear();
}
DiskAnnMetaHeader(const DiskAnnMetaHeader &header) {
memcpy(this, &header, sizeof(header));
}
DiskAnnMetaHeader &operator=(const DiskAnnMetaHeader &header) {
if (this != &header) {
memcpy(this, &header, sizeof(header));
}
return *this;
}
void reset() {
doc_cnt = 0U;
}
void clear() {
memset(this, 0, sizeof(DiskAnnMetaHeader));
}
};
struct DiskAnnPqMeta {
public:
uint64_t full_pivot_data_size{0};
uint64_t centroid_data_size{0};
uint64_t chunk_offsets_size{0};
uint64_t chunk_num{0};
uint8_t reserved[128];
DiskAnnPqMeta() {
clear();
}
DiskAnnPqMeta(const DiskAnnPqMeta &meta) {
memcpy(this, &meta, sizeof(meta));
}
DiskAnnPqMeta &operator=(const DiskAnnPqMeta &meta) {
memcpy(this, &meta, sizeof(meta));
return *this;
}
void clear() {
memset(this, 0, sizeof(DiskAnnPqMeta));
}
};
static_assert(sizeof(DiskAnnMetaHeader) == 4096,
"DiskAnnMetaHeader size must stay 4096 bytes (on-disk format)");
static_assert(sizeof(DiskAnnPqMeta) % 32 == 0,
"DiskAnnPqMeta size must be a multiple of 32 bytes");
class DiskAnnEntity {
public:
DiskAnnEntity() = default;
virtual ~DiskAnnEntity() = default;
//! Constructor
DiskAnnEntity(const DiskAnnMetaHeader &meta_header,
const DiskAnnPqMeta &pq_meta) {
meta_header_ = meta_header;
pq_meta_ = pq_meta;
}
//! DiskAnnEntity Pointerd;
typedef std::shared_ptr<DiskAnnEntity> Pointer;
public:
static size_t AlignSize(size_t size) {
return (size + 0xFFF) & (~0xFFF);
}
public:
virtual int add_vector(diskann_key_t /*key*/, const void * /*vec*/) {
return IndexError_NotImplemented;
}
virtual const void *get_vector(diskann_id_t /*id*/) const {
return nullptr;
}
virtual std::pair<uint32_t, const diskann_id_t *> get_neighbors(
diskann_id_t /*id*/) const {
return std::make_pair(0, nullptr);
}
virtual int set_neighbors(
diskann_id_t /*id*/, const std::vector<diskann_id_t> & /*neighbor_ids*/) {
return IndexError_NotImplemented;
}
virtual int add_neighbor(diskann_id_t /*id*/, diskann_id_t /*neighbor_id*/) {
return IndexError_NotImplemented;
}
//! Get node id of primary key
virtual diskann_id_t get_id(diskann_key_t key) const = 0;
//! Get primary key of the node id
virtual diskann_key_t get_key(diskann_id_t id) const = 0;
public:
uint64_t max_node_size() const {
return meta_header_.max_node_size;
}
uint64_t medoid() const {
return meta_header_.medoid;
}
uint64_t *mutable_medoid() {
return &meta_header_.medoid;
}
uint64_t node_per_sector() const {
return meta_header_.node_per_sector;
}
uint64_t pq_chunk_num() {
return pq_meta_.chunk_num;
}
uint64_t doc_cnt() const {
return meta_header_.doc_cnt;
}
uint64_t *mutable_doc_cnt() {
return &meta_header_.doc_cnt;
}
uint64_t max_degree() {
return meta_header_.max_degree;
}
DiskAnnPqMeta *mutable_pq_meta() {
return &pq_meta_;
}
public:
virtual const DiskAnnEntity::Pointer clone() const {
LOG_ERROR("Update neighbors not implemented");
return DiskAnnEntity::Pointer();
}
public:
const static std::string kDiskAnnVectorSegmentId;
const static std::string kDiskAnnMetaSegmentId;
const static std::string kDiskAnnPqMetaSegmentId;
const static std::string kDiskAnnPqDataSegmentId;
const static std::string kDiskAnnDummpySegmentId;
const static std::string kDiskAnnMappingSegmentId;
const static std::string kDiskAnnKeyMappingSegmentId;
const static std::string kDiskAnnEntryPointSegmentId;
const static std::string kDiskAnnKeySegmentId;
constexpr static float kDefaultBFNegativeProbility = 0.001f;
constexpr static float kDefaultGraphSlackFactor = 1.3f;
constexpr static float kDefaultAlpha = 1.2f;
constexpr static uint32_t kDefaultMaxOcclusionSize = 750;
constexpr static uint32_t kDefaultMaxDegree = 100;
constexpr static uint32_t kDefaultCompressBatchSize = 5000000;
constexpr static uint32_t kRevision = 0U;
protected:
DiskAnnMetaHeader meta_header_;
DiskAnnPqMeta pq_meta_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,299 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_file_reader.h"
#include <cassert>
#include <cerrno>
#include <cstdio>
#include <cstring>
#include <iostream>
#include <zvec/core/framework/index_logger.h>
#define MAX_EVENTS 1024
namespace zvec {
namespace core {
#if (defined(__linux) || defined(__linux__))
typedef struct io_event io_event_t;
typedef struct iocb iocb_t;
#endif
int setup_io_ctx(IOContext &ctx) {
#if (defined(__linux) || defined(__linux__))
int ret = io_setup(MAX_EVENTS, &ctx);
return ret;
#else
return 0;
#endif
}
int destroy_io_ctx(IOContext &ctx) {
#if (defined(__linux) || defined(__linux__))
int ret = io_destroy(ctx);
return ret;
#else
return 0;
#endif
}
static int execute_io_pread(int fd, std::vector<AlignedRead> &read_reqs) {
for (auto &req : read_reqs) {
ssize_t bytes_read = ::pread(fd, req.buf, req.len, req.offset);
if (bytes_read < 0) {
LOG_ERROR("pread failed; errno=%d, %s, offset=%lu, len=%lu", errno,
::strerror(errno), (unsigned long)req.offset,
(unsigned long)req.len);
return IndexError_Runtime;
}
if ((size_t)bytes_read != req.len) {
LOG_ERROR("pread short read; got=%zd, expected=%lu", bytes_read,
(unsigned long)req.len);
return IndexError_Runtime;
}
}
return 0;
}
int execute_io(IOContext ctx, int fd, std::vector<AlignedRead> &read_reqs,
uint64_t n_retries = 0) {
#if (defined(__linux) || defined(__linux__))
uint64_t iters = DiskAnnUtil::div_round_up(read_reqs.size(), MAX_EVENTS);
for (uint64_t iter = 0; iter < iters; iter++) {
uint64_t n_ops = std::min((uint64_t)read_reqs.size() - (iter * MAX_EVENTS),
(uint64_t)MAX_EVENTS);
std::vector<iocb_t *> cbs(n_ops, nullptr);
std::vector<io_event_t> evts(n_ops);
std::vector<struct iocb> cb(n_ops);
for (uint64_t j = 0; j < n_ops; j++) {
io_prep_pread(cb.data() + j, fd, read_reqs[j + iter * MAX_EVENTS].buf,
read_reqs[j + iter * MAX_EVENTS].len,
read_reqs[j + iter * MAX_EVENTS].offset);
}
for (uint64_t i = 0; i < n_ops; i++) {
cbs[i] = cb.data() + i;
}
size_t n_tries = 0;
// Phase 1: io_submit with retry.
while (true) {
int ret = io_submit(ctx, (int64_t)n_ops, cbs.data());
if (ret == (int)n_ops) {
break;
}
if ((ret == -EAGAIN || ret == -EINTR) && n_tries < n_retries) {
n_tries++;
continue;
}
LOG_WARN(
"io_submit failed; returned: %d, expected=%lu. falling back to "
"pread",
ret, n_ops);
return execute_io_pread(fd, read_reqs);
}
// Phase 2: io_getevents with retry (never re-submits).
n_tries = 0;
while (true) {
int ret = io_getevents(ctx, (int64_t)n_ops, (int64_t)n_ops, evts.data(),
nullptr);
if (ret == (int)n_ops) {
break;
}
if (ret == -EINTR && n_tries < n_retries) {
n_tries++;
continue;
}
LOG_WARN(
"io_getevents failed; returned: %d, expected=%lu, errno=%d, %s, "
"falling back to pread",
ret, n_ops, errno, ::strerror(-ret));
return execute_io_pread(fd, read_reqs);
}
// Phase 3: verify each completed read (res must equal requested length).
bool all_ok = true;
for (uint64_t i = 0; i < n_ops; i++) {
int64_t expected_len = read_reqs[i + iter * MAX_EVENTS].len;
if ((int64_t)evts[i].res != expected_len) {
LOG_WARN("aio request %zu failed: res=%ld, expected=%ld, offset=%zu",
(size_t)i, (long)evts[i].res, (long)expected_len,
(size_t)read_reqs[i + iter * MAX_EVENTS].offset);
all_ok = false;
}
}
if (!all_ok) {
return execute_io_pread(fd, read_reqs);
}
}
return 0;
#else
return execute_io_pread(fd, read_reqs);
#endif
}
LinuxAlignedFileReader::LinuxAlignedFileReader(int file_desc) {
this->file_desc = file_desc;
}
LinuxAlignedFileReader::LinuxAlignedFileReader() {
this->file_desc = -1;
}
LinuxAlignedFileReader::~LinuxAlignedFileReader() {
deregister_all_threads();
if (file_desc >= 0) {
::close(file_desc);
file_desc = -1;
}
}
IOContext &LinuxAlignedFileReader::get_ctx() {
std::unique_lock<std::mutex> lk(ctx_mut);
auto it = ctx_map.find(std::this_thread::get_id());
if (it == ctx_map.end()) {
LOG_ERROR("bad thread access; returning -1 as io_context_t");
return this->bad_ctx;
} else {
return it->second;
}
}
void LinuxAlignedFileReader::register_thread() {
#if (defined(__linux) || defined(__linux__))
auto thread_id = std::this_thread::get_id();
std::unique_lock<std::mutex> lk(ctx_mut);
if (ctx_map.find(thread_id) != ctx_map.end()) {
LOG_ERROR("multiple calls to register_thread from the same thread");
return;
}
IOContext ctx = nullptr;
int ret = io_setup(MAX_EVENTS, &ctx);
if (ret != 0) {
lk.unlock();
if (ret == -EAGAIN) {
LOG_ERROR(
"io_setup failed with EAGAIN: Consider increasing "
"/proc/sys/fs/aio-max-nr");
} else {
LOG_ERROR("io_setup failed; returned: %d, %s", ret, ::strerror(-ret));
;
}
} else {
LOG_INFO("allocating ctx: %lu", (uint64_t)ctx);
ctx_map[thread_id] = ctx;
}
lk.unlock();
#endif
}
void LinuxAlignedFileReader::deregister_thread() {
#if (defined(__linux) || defined(__linux__))
auto thread_id = std::this_thread::get_id();
IOContext ctx;
{
std::lock_guard<std::mutex> lk(ctx_mut);
auto it = ctx_map.find(thread_id);
if (it == ctx_map.end()) {
LOG_ERROR("deregister_thread: thread not registered");
return;
}
ctx = it->second;
ctx_map.erase(it);
}
// io_destroy is a syscall; keep it outside the lock to avoid blocking others
io_destroy(ctx);
LOG_INFO("returned ctx from thread");
#endif
}
void LinuxAlignedFileReader::deregister_all_threads() {
#if (defined(__linux) || defined(__linux__))
std::unique_lock<std::mutex> lk(ctx_mut);
for (auto x = ctx_map.begin(); x != ctx_map.end(); x++) {
IOContext ctx = x->second;
io_destroy(ctx);
}
ctx_map.clear();
#endif
}
void LinuxAlignedFileReader::open(const std::string &fname) {
int flags = O_RDONLY;
#if defined(__linux__) || defined(__linux)
flags |= O_DIRECT | O_LARGEFILE;
#endif
this->file_desc = ::open(fname.c_str(), flags);
#if defined(__linux__) || defined(__linux)
// O_DIRECT may not be supported on all filesystems (e.g. tmpfs, overlay).
// Fall back to regular buffered I/O when it fails.
if (this->file_desc == -1) {
LOG_WARN(
"open with O_DIRECT failed for %s (errno=%d: %s), "
"falling back to buffered I/O",
fname.c_str(), errno, ::strerror(errno));
this->file_desc = ::open(fname.c_str(), O_RDONLY | O_LARGEFILE);
}
#endif
if (this->file_desc == -1) {
LOG_ERROR("Failed to open file: %s (errno=%d: %s)", fname.c_str(), errno,
::strerror(errno));
}
LOG_INFO("Opened file : %s", fname.c_str());
}
void LinuxAlignedFileReader::close() {
if (file_desc >= 0) {
::close(file_desc);
file_desc = -1;
}
}
int LinuxAlignedFileReader::read(std::vector<AlignedRead> &read_reqs,
IOContext &ctx, bool async) {
if (async == true) {
LOG_WARN("Async currently not supported");
}
if (this->file_desc == -1) {
LOG_ERROR("Attempt to read from invalid file descriptor");
return IndexError_Runtime;
}
int ret = execute_io(ctx, this->file_desc, read_reqs);
return ret;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,104 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#define MAX_IO_DEPTH 128
#include <fcntl.h>
#if (defined(__linux) || defined(__linux__))
#include <libaio.h>
#endif
#include <unistd.h>
#include <atomic>
#include <vector>
#include <zvec/core/framework/index_context.h>
#include "diskann_util.h"
namespace zvec {
namespace core {
#if (defined(__linux) || defined(__linux__))
typedef io_context_t IOContext;
#else
typedef uint32_t IOContext;
#endif
int setup_io_ctx(IOContext &ctx);
int destroy_io_ctx(IOContext &ctx);
struct AlignedRead {
uint64_t offset;
uint64_t len;
void *buf;
AlignedRead() : offset(0), len(0), buf(nullptr) {}
AlignedRead(uint64_t offset, uint64_t len, void *buf)
: offset(offset), len(len), buf(buf) {
ailego_assert(static_cast<size_t>(offset) % 512 == 0);
ailego_assert(static_cast<size_t>(len) % 512 == 0);
ailego_assert(reinterpret_cast<size_t>(buf) % 512 == 0);
}
};
class AlignedFileReader {
protected:
std::map<std::thread::id, IOContext> ctx_map;
std::mutex ctx_mut;
public:
virtual IOContext &get_ctx() = 0;
virtual ~AlignedFileReader() {};
virtual void register_thread() = 0;
virtual void deregister_thread() = 0;
virtual void deregister_all_threads() = 0;
virtual void open(const std::string &fname) = 0;
virtual void close() = 0;
virtual int read(std::vector<AlignedRead> &read_reqs, IOContext &ctx,
bool async = false) = 0;
};
class LinuxAlignedFileReader : public AlignedFileReader {
private:
uint64_t file_sz;
int file_desc;
IOContext bad_ctx = (IOContext)-1;
public:
LinuxAlignedFileReader();
LinuxAlignedFileReader(int file_desc);
~LinuxAlignedFileReader();
public:
IOContext &get_ctx();
void register_thread();
void deregister_thread();
void deregister_all_threads();
void open(const std::string &fname);
void close();
int read(std::vector<AlignedRead> &read_reqs, IOContext &ctx,
bool async = false);
};
} // namespace core
} // namespace zvec
+325
View File
@@ -0,0 +1,325 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <fstream>
#include <zvec/core/framework/index_holder.h>
#include "diskann_entity.h"
namespace zvec {
namespace core {
struct DiskAnnIndexHolderMeta {
uint32_t element_size_;
uint32_t key_size_;
uint32_t sector_size_;
uint32_t doc_cnt_;
uint8_t reserve_[];
};
class DiskAnnIndexHolder : public IndexHolder {
public:
typedef std::shared_ptr<DiskAnnIndexHolder> Pointer;
public:
enum Status { STATUS_UNINITED = 0, STATUS_WRITE = 1, STATUS_READ = 2 };
public:
static constexpr uint32_t kDataSectorSize = 128 * 1024;
static constexpr uint32_t kMetaSectorSize = 4096;
public:
inline static uint32_t get_sector_id(uint32_t id, uint32_t sector_vec_num) {
return id / sector_vec_num;
}
inline static uint32_t get_sector_offset(uint32_t id, uint32_t sector_vec_num,
uint32_t data_size) {
return (id % sector_vec_num) * data_size;
}
public:
/*! Random Access Index Holder Iterator
*/
class Iterator : public IndexHolder::Iterator {
public:
//! Index Holder Iterator Pointer
typedef std::unique_ptr<Iterator> Pointer;
//! Constructor
Iterator(DiskAnnIndexHolder *owner)
: holder_(owner), sector_id_{0}, sector_offset_{0} {
path_ = holder_->path();
data_size_ = holder_->data_size();
data_sector_size_ = holder_->data_sector_size();
meta_sector_size_ = holder_->meta_sector_size();
sector_buffer_.resize(data_sector_size_);
sector_vec_num_ = data_sector_size_ / data_size_;
}
//! Destructor
virtual ~Iterator(void) {
if (file_.is_open()) {
file_.close();
}
}
int init() {
file_.open(path_, std::ios::in);
if (!file_.is_open()) {
LOG_ERROR("file can not create, %s", path_.c_str());
return IndexError_OpenFile;
}
file_.seekg(meta_sector_size_);
read_sector();
return 0;
}
//! Retrieve pointer of data
const void *data(void) const override {
const uint8_t *data_ptr =
reinterpret_cast<const uint8_t *>(sector_buffer_.data());
return data_ptr + sector_offset_ + sizeof(diskann_key_t);
}
//! Test if the iterator is valid
bool is_valid(void) const override {
return id_ < holder_->count();
}
//! Retrieve primary key
uint64_t key(void) const override {
const uint8_t *data_ptr =
reinterpret_cast<const uint8_t *>(sector_buffer_.data());
uint64_t key =
*reinterpret_cast<const uint64_t *>(data_ptr + sector_offset_);
return key;
}
//! Next iterator
void next(void) override {
++id_;
uint32_t sector_id = get_sector_id(id_, sector_vec_num_);
if (sector_id > sector_id_) {
file_.seekg(sector_id * data_sector_size_ + meta_sector_size_);
read_sector();
sector_id_ = sector_id;
}
sector_offset_ = get_sector_offset(id_, sector_vec_num_, data_size_);
}
int read_sector() {
file_.read(&((sector_buffer_)[0]), data_sector_size_);
if (!file_) {
LOG_ERROR("Failed to read sector from file: %s", path_.c_str());
return IndexError_ReadData;
}
return 0;
}
private:
//! Members
DiskAnnIndexHolder *holder_{nullptr};
std::string path_;
std::ifstream file_;
uint32_t sector_id_{0};
std::string sector_buffer_;
uint32_t id_{0};
uint32_t data_size_{0};
uint32_t sector_offset_{0};
uint32_t data_sector_size_{0};
uint32_t meta_sector_size_{0};
uint32_t sector_vec_num_{0};
};
public:
DiskAnnIndexHolder(IndexMeta &meta, std::string &path) {
path_ = path;
data_size_ = meta.element_size() + sizeof(diskann_key_t);
dimension_ = meta.dimension();
type_ = meta.data_type();
element_size_ = meta.element_size();
sector_vec_num_ = data_sector_size_ / data_size_;
padding_size_ = data_sector_size_ - sector_vec_num_ * data_size_;
sector_buffer_.resize(data_sector_size_);
sector_internal_id_ = 0;
}
~DiskAnnIndexHolder() override {
if (file_.is_open()) {
file_.close();
}
}
//! Init
int init() {
file_.open(path_, std::ios::out | std::ios::trunc);
if (!file_.is_open()) {
LOG_ERROR("file can not create, %s", path_.c_str());
return IndexError_OpenFile;
}
DiskAnnIndexHolderMeta holder_meta;
holder_meta.element_size_ = element_size_;
holder_meta.key_size_ = sizeof(diskann_key_t);
holder_meta.sector_size_ = data_sector_size_;
std::vector<uint8_t> empty_sector;
empty_sector.resize(meta_sector_size_);
std::memset(&(empty_sector[0]), 0, meta_sector_size_);
std::memcpy(&(empty_sector[0]), &holder_meta,
sizeof(DiskAnnIndexHolderMeta));
file_.write(reinterpret_cast<const char *>(&(empty_sector[0])),
meta_sector_size_);
if (!file_) {
LOG_ERROR("Failed to write meta sector to file: %s", path_.c_str());
return IndexError_WriteData;
}
status_ = STATUS_WRITE;
return 0;
}
int close() {
if (sector_internal_id_ != 0) {
file_.write(reinterpret_cast<const char *>(&(sector_buffer_[0])),
data_sector_size_);
}
file_.close();
return 0;
}
//! Retrieve count of elements in holder (-1 indicates unknown)
size_t count(void) const override {
return count_;
}
//! Retrieve dimension
size_t dimension(void) const override {
return dimension_;
}
//! Retrieve type information
IndexMeta::DataType data_type(void) const override {
return type_;
}
//! Retrieve element size in bytes
size_t element_size(void) const override {
return element_size_;
}
//! Retrieve if it can multi-pass
bool multipass(void) const override {
return true;
}
//! Create a new iterator
IndexHolder::Iterator::Pointer create_iterator(void) override {
auto pointer = std::make_unique<DiskAnnIndexHolder::Iterator>(this);
if (pointer->init() != 0) {
return nullptr;
}
return pointer;
}
int emplace(uint64_t pkey, const void *vec) {
if (status_ != STATUS_WRITE) {
return IndexError_NoReady;
}
uint8_t *data_ptr = reinterpret_cast<uint8_t *>(&(sector_buffer_[0])) +
sector_internal_id_ * data_size_;
std::memcpy(data_ptr, &pkey, sizeof(diskann_key_t));
std::memcpy(data_ptr + sizeof(diskann_key_t), vec, element_size_);
sector_internal_id_++;
if (sector_internal_id_ >= sector_vec_num_) {
std::memset(data_ptr + data_size_, 0, padding_size_);
file_.write(reinterpret_cast<const char *>(&(sector_buffer_[0])),
data_sector_size_);
sector_internal_id_ = 0;
sector_id_++;
}
count_++;
return 0;
}
uint32_t data_sector_size() {
return data_sector_size_;
}
uint32_t meta_sector_size() {
return meta_sector_size_;
}
uint32_t data_size() {
return data_size_;
}
uint32_t *mutable_sector_id() {
return &sector_id_;
}
uint32_t sector_id() {
return sector_id_;
}
std::string &path() {
return path_;
}
private:
std::string path_;
std::ofstream file_;
uint32_t element_size_{0};
uint32_t dimension_{0};
IndexMeta::DataType type_{IndexMeta::DataType::DT_UNDEFINED};
uint32_t sector_vec_num_{0};
uint32_t data_size_{0};
uint32_t padding_size_{0};
uint32_t meta_sector_size_{DiskAnnIndexHolder::kMetaSectorSize};
uint32_t data_sector_size_{DiskAnnIndexHolder::kDataSectorSize};
std::string sector_buffer_;
uint32_t sector_internal_id_{0};
uint32_t sector_id_{0};
uint32_t count_{0};
uint32_t status_{STATUS_UNINITED};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,120 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/core/framework/index_provider.h>
#include <zvec/core/framework/index_searcher.h>
#include <zvec/core/framework/index_streamer.h>
#include "diskann_entity.h"
namespace zvec {
namespace core {
//! IndexProvider implementation backed by a DiskAnn entity.
//!
//! Used by ``MixedStreamerReducer`` during segment merge: the reducer needs
//! to walk every vector held by a source DiskAnn streamer and feed it into
//! the merge target. Vectors are read on demand from the entity's on-disk
//! vector segment via ``DiskAnnEntity::get_vector(id)``.
class DiskAnnIndexProvider : public IndexProvider {
public:
DiskAnnIndexProvider(const IndexMeta &meta,
const DiskAnnEntity::Pointer &entity,
const std::string &owner)
: meta_(meta), entity_(entity), owner_class_(owner) {}
DiskAnnIndexProvider(const DiskAnnIndexProvider &) = delete;
DiskAnnIndexProvider &operator=(const DiskAnnIndexProvider &) = delete;
public:
IndexProvider::Iterator::Pointer create_iterator() override {
return IndexProvider::Iterator::Pointer(new (std::nothrow)
Iterator(entity_));
}
size_t count(void) const override {
return entity_->doc_cnt();
}
size_t dimension(void) const override {
return meta_.dimension();
}
IndexMeta::DataType data_type(void) const override {
return meta_.data_type();
}
size_t element_size(void) const override {
return meta_.element_size();
}
const void *get_vector(uint64_t key) const override {
diskann_id_t id = entity_->get_id(static_cast<diskann_key_t>(key));
if (id == kInvalidId) {
return nullptr;
}
return entity_->get_vector(id);
}
const std::string &owner_class(void) const override {
return owner_class_;
}
private:
class Iterator : public IndexProvider::Iterator {
public:
explicit Iterator(const DiskAnnEntity::Pointer &entity)
: entity_(entity), cur_id_(0U) {
cur_id_ = next_valid_id(0U);
}
const void *data(void) const override {
return entity_->get_vector(cur_id_);
}
bool is_valid(void) const override {
return cur_id_ < static_cast<diskann_id_t>(entity_->doc_cnt());
}
uint64_t key(void) const override {
return static_cast<uint64_t>(entity_->get_key(cur_id_));
}
void next(void) override {
cur_id_ = next_valid_id(cur_id_ + 1);
}
private:
//! Skip ids that map to ``kInvalidKey`` (deleted / never populated slots).
diskann_id_t next_valid_id(diskann_id_t start_id) const {
const auto total = static_cast<diskann_id_t>(entity_->doc_cnt());
for (diskann_id_t i = start_id; i < total; ++i) {
if (entity_->get_key(i) != kInvalidKey) {
return i;
}
}
return total;
}
DiskAnnEntity::Pointer entity_;
diskann_id_t cur_id_;
};
IndexMeta meta_;
DiskAnnEntity::Pointer entity_;
std::string owner_class_;
};
} // namespace core
} // namespace zvec
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,109 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <cstdint>
#include <zvec/core/framework/index_framework.h>
#include "diskann_context.h"
#include "diskann_file_reader.h"
#include "diskann_pq_table.h"
#include "diskann_searcher_entity.h"
#include "diskann_util.h"
namespace zvec {
namespace core {
class DiskAnnIndexer {
public:
typedef std::shared_ptr<DiskAnnIndexer> Pointer;
public:
DiskAnnIndexer(const IndexMeta &meta);
~DiskAnnIndexer();
public:
int init(DiskAnnSearcherEntity &entity);
int load_cache_list(const std::vector<diskann_id_t> &node_list);
void cache_bfs_levels(uint64_t num_nodes_to_cache,
std::vector<diskann_id_t> &node_list);
int cached_beam_search(DiskAnnContext *ctx);
int cached_beam_search_by_group(DiskAnnContext *ctx);
int cached_beam_search_in_mem(DiskAnnContext *ctx);
int knn_search(DiskAnnContext *ctx);
int linear_search(DiskAnnContext *ctx);
int keys_search(const std::vector<diskann_key_t> &keys, DiskAnnContext *ctx);
int get_vector(diskann_id_t id, IndexContext::Pointer &context,
std::string &vector);
diskann_key_t get_key(diskann_id_t id) const;
diskann_id_t get_id(diskann_key_t key) const;
//! Copy element_size() bytes from src into a new vector value string
std::string make_vector_copy(const void *src) const {
return std::string(static_cast<const char *>(src), meta_.element_size());
}
std::vector<bool> read_nodes(
const std::vector<diskann_id_t> &node_ids,
std::vector<void *> &coord_buffers,
std::vector<std::pair<uint32_t, diskann_id_t *>> &nbr_buffers);
protected:
int use_medroids_data_as_centroids();
private:
DiskAnnSearcherEntity *entity_;
IndexStorage::Pointer storage_{};
IndexMeta meta_;
uint32_t max_degree_{0};
uint32_t node_per_sector_{0};
uint32_t max_node_size_{0};
uint64_t pq_chunk_num_{0};
uint64_t disk_bytes_per_point_{0};
uint64_t aligned_dim_{0};
uint64_t index_segment_offset_{0};
uint64_t sector_num_per_node_{0};
float *centroid_data_{nullptr};
diskann_id_t medoid_;
std::vector<diskann_id_t> entrypoints_;
std::shared_ptr<LinuxAlignedFileReader> reader_{nullptr};
PQTable::Pointer pq_table_;
IOContext init_ctx_{0};
std::vector<diskann_id_t> neighbor_cache_buffer_;
void *coord_cache_buf_{nullptr};
std::map<diskann_id_t, void *> coord_cache_;
std::map<diskann_id_t, std::pair<uint32_t, diskann_id_t *>> neighbor_cache_;
uint32_t beam_width_{2};
uint32_t io_limit_{std::numeric_limits<uint32_t>::max()};
uint64_t doc_cnt_{0};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,53 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
namespace zvec {
namespace core {
static const std::string PARAM_DISKANN_BUILDER_MAX_DEGREE(
"zvec.diskann.builder.max_degree");
static const std::string PARAM_DISKANN_BUILDER_LIST_SIZE(
"zvec.diskann.builder.list_size");
static const std::string PARAM_DISKANN_BUILDER_MEMORY_LIMIT(
"zvec.diskann.builder.memory_limit");
static const std::string PARAM_DISKANN_BUILDER_MEMORY_BUDGET(
"zvec.diskann.builder.memory_budget");
static const std::string PARAM_DISKANN_BUILDER_DISK_PQ_DIM(
"zvec.diskann.builder.disk_pq_dim");
static const std::string PARAM_DISKANN_BUILDER_THREAD_COUNT(
"zvec.diskann.builder.thread_count");
static const std::string PARAM_DISKANN_BUILDER_MAX_TRAIN_SAMPLE_COUNT(
"zvec.diskann.builder.max_train_sample_count");
static const std::string PARAM_DISKANN_BUILDER_TRAIN_SAMPLE_RATIO(
"zvec.diskann.builder.train_sample_ratio");
static const std::string PARAM_DISKANN_BUILDER_MAX_PQ_CHUNK_NUM(
"zvec.diskann.builder.max_pq_chunk_num");
static const std::string PARAM_DISKANN_SEARCHER_LIST_SIZE(
"zvec.diskann.searcher.list_size");
static const std::string PARAM_DISKANN_SEARCHER_CACHE_NODE_NUM(
"zvec.diskann.searcher.cache_node_num");
static const std::string PARAM_DISKANN_REDUCER_INDEX_NAME(
"zvec.diskann.reducer.index_name");
static const std::string PARAM_DISKANN_REDUCER_WORKING_PATH(
"zvec.diskann.reducer.working_path");
static const std::string PARAM_DISKANN_REDUCER_NUM_OF_ADD_THREADS(
"zvec.diskann.reducer.num_of_add_threads");
} // namespace core
} // namespace zvec
@@ -0,0 +1,117 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_pq_table.h"
#include "diskann_entity.h"
namespace zvec {
namespace core {
PQTable::PQTable(const IndexMeta &meta, uint32_t chunk_num)
: chunk_num_(chunk_num) {
meta_ = meta;
if (meta.metric_name() == "Cosine") {
if (meta.data_type() == IndexMeta::DataType::DT_FP32) {
meta_.set_dimension(meta.dimension() - 1);
} else {
meta_.set_dimension(meta.dimension() - 2);
}
}
}
PQTable::~PQTable() {}
int PQTable::init(std::vector<uint8_t> &full_pivot_data,
std::vector<uint8_t> &centroid,
std::vector<uint32_t> &chunk_offsets,
std::vector<uint8_t> &pq_data) {
full_pivot_data_ = std::move(full_pivot_data);
centroid_ = std::move(centroid);
chunk_offsets_ = std::move(chunk_offsets);
pq_data_ = std::move(pq_data);
// alloc and compute transpose
transposed_tables_.resize(kPQCentroidNum * meta_.element_size());
uint32_t dim = meta_.dimension();
uint32_t type = meta_.data_type();
switch (type) {
case IndexMeta::DataType::DT_FP32: {
float *transposed_tables_ptr =
reinterpret_cast<float *>(&transposed_tables_[0]);
float *full_pivot_data_ptr =
reinterpret_cast<float *>(&full_pivot_data_[0]);
for (size_t i = 0; i < kPQCentroidNum; i++) {
for (size_t j = 0; j < dim; j++) {
transposed_tables_ptr[j * kPQCentroidNum + i] =
full_pivot_data_ptr[i * dim + j];
}
}
break;
}
case IndexMeta::DataType::DT_FP16: {
ailego::Float16 *transposed_tables_ptr =
reinterpret_cast<ailego::Float16 *>(&transposed_tables_[0]);
ailego::Float16 *full_pivot_data_ptr =
reinterpret_cast<ailego::Float16 *>(&full_pivot_data_[0]);
for (size_t i = 0; i < kPQCentroidNum; i++) {
for (size_t j = 0; j < dim; j++) {
transposed_tables_ptr[j * kPQCentroidNum + i] =
full_pivot_data_ptr[i * dim + j];
}
}
break;
}
default:
LOG_ERROR("unsupported type, type: %u", type);
return IndexError_Unsupported;
}
return 0;
}
void PQTable::aggregate_coords(uint32_t id_num, const diskann_id_t *ids,
const uint8_t *all_coords, size_t dim,
uint8_t *out) {
for (size_t i = 0; i < id_num; i++) {
memcpy(out + i * dim, all_coords + ids[i] * dim, dim * sizeof(uint8_t));
}
}
void PQTable::pq_dist_lookup(const uint8_t *pq_ids, size_t id_num,
size_t pq_nchunks, const float *pq_dist_buffer,
float *dists_out) {
ailego_prefetch(dists_out);
ailego_prefetch(pq_ids);
ailego_prefetch(pq_ids + 64);
ailego_prefetch(pq_ids + 128);
memset(dists_out, 0, id_num * sizeof(float));
for (size_t chunk = 0; chunk < pq_nchunks; chunk++) {
const float *chunk_dists = pq_dist_buffer + kPQCentroidNum * chunk;
if (chunk < pq_nchunks - 1) {
ailego_prefetch(chunk_dists + kPQCentroidNum);
}
for (size_t idx = 0; idx < id_num; idx++) {
uint8_t pq_centerid = pq_ids[pq_nchunks * idx + chunk];
dists_out[idx] += chunk_dists[pq_centerid];
}
}
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,154 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/core/framework/index_framework.h>
#include "diskann_entity.h"
namespace zvec {
namespace core {
class PQTable {
public:
typedef std::shared_ptr<PQTable> Pointer;
public:
static constexpr uint32_t kPQBitNum = 8;
static constexpr uint32_t kPQCentroidNum = 1 << kPQBitNum;
static constexpr uint32_t kMaxTrainSampleCount = 200000;
static constexpr double kTrainSampleRatio = 1.0;
static constexpr uint32_t kMeanIterNum = 12;
public:
PQTable(const IndexMeta &meta, uint32_t chunk_num);
virtual ~PQTable();
int init(std::vector<uint8_t> &table, std::vector<uint8_t> &centroid,
std::vector<uint32_t> &chunk_offsets, std::vector<uint8_t> &pq_data);
template <typename T>
void compute_distance_table_ip(const T *query_vec, float *dist_vec) {
memset(dist_vec, 0, kPQCentroidNum * chunk_num_ * sizeof(float));
const T *transposed_tables_ptr =
reinterpret_cast<const T *>(transposed_tables_.data());
// chunk wise distance computation
for (size_t chunk = 0; chunk < chunk_num_; chunk++) {
// sum (q-c)^2 for the dimensions associated with this chunk
float *chunk_dists = dist_vec + (kPQCentroidNum * chunk);
for (size_t j = chunk_offsets_[chunk]; j < chunk_offsets_[chunk + 1];
j++) {
const T *centers_dim_vec = &transposed_tables_ptr[kPQCentroidNum * j];
for (size_t idx = 0; idx < kPQCentroidNum; idx++) {
float centor_data = centers_dim_vec[idx];
float query_data = query_vec[j];
float dim_score = centor_data * query_data;
chunk_dists[idx] += -dim_score;
}
}
}
}
template <typename T>
void compute_distance_table(const T *query_vec, float *dist_vec) {
memset(dist_vec, 0, kPQCentroidNum * chunk_num_ * sizeof(float));
const T *transposed_tables_ptr =
reinterpret_cast<const T *>(transposed_tables_.data());
// chunk wise distance computation
for (size_t chunk = 0; chunk < chunk_num_; chunk++) {
// sum (q-c)^2 for the dimensions associated with this chunk
float *chunk_dists = dist_vec + (kPQCentroidNum * chunk);
for (size_t j = chunk_offsets_[chunk]; j < chunk_offsets_[chunk + 1];
j++) {
const T *centers_dim_vec = &transposed_tables_ptr[kPQCentroidNum * j];
for (size_t idx = 0; idx < kPQCentroidNum; idx++) {
float diff = centers_dim_vec[idx] - query_vec[j];
chunk_dists[idx] += (diff * diff);
}
}
}
}
template <typename T>
void preprocess_query(T *query) {
const T *centroid_ptr = reinterpret_cast<const T *>(centroid_.data());
for (size_t i = 0; i < meta_.dimension(); i++) {
query[i] -= centroid_ptr[i];
}
}
void aggregate_coords(uint32_t id_num, const diskann_id_t *ids,
const uint8_t *all_coords, size_t dim, uint8_t *out);
void pq_dist_lookup(const uint8_t *pq_ids, size_t id_num, size_t pq_nchunks,
const float *pq_dist_buffer, float *dists);
void compute_dists(uint32_t id_num, const diskann_id_t *ids,
uint32_t chunk_num, float *pq_dist_buffer,
void *coord_buffer, float *dists) {
uint8_t *coord_buffer_ptr = reinterpret_cast<uint8_t *>(coord_buffer);
aggregate_coords(id_num, ids, this->pq_data(), chunk_num, coord_buffer_ptr);
pq_dist_lookup(coord_buffer_ptr, id_num, chunk_num, pq_dist_buffer, dists);
return;
}
int preprocess_pq_dist_table(void *query_rotated, float *dist_buffer) {
switch (meta_.data_type()) {
case IndexMeta::DataType::DT_FP32:
preprocess_query(reinterpret_cast<float *>(query_rotated));
compute_distance_table(reinterpret_cast<float *>(query_rotated),
dist_buffer);
break;
case IndexMeta::DataType::DT_FP16:
preprocess_query(reinterpret_cast<ailego::Float16 *>(query_rotated));
compute_distance_table(
reinterpret_cast<ailego::Float16 *>(query_rotated), dist_buffer);
break;
default:
LOG_ERROR("Unsupported Type: %u", meta_.data_type());
return IndexError_Unsupported;
}
return 0;
}
public:
const uint8_t *pq_data() const {
return pq_data_.data();
}
private:
std::vector<uint8_t> full_pivot_data_;
std::vector<uint8_t> transposed_tables_;
std::vector<uint8_t> centroid_;
std::vector<uint32_t> chunk_offsets_;
std::vector<uint8_t> pq_data_;
IndexMeta meta_;
uint64_t chunk_num_{0};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,417 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_pq_trainer.h"
#include "diskann_entity.h"
#include "diskann_util.h"
namespace zvec {
namespace core {
DiskAnnPqTrainer::DiskAnnPqTrainer(uint32_t max_train_sample_count)
: max_train_sample_count_{max_train_sample_count} {}
DiskAnnPqTrainer::~DiskAnnPqTrainer() {}
int DiskAnnPqTrainer::gen_random_sample(IndexHolder::Pointer holder,
const IndexMeta &meta,
std::string &sample_data,
size_t &sample_size) {
double train_sample_ratio =
max_train_sample_count_ < 1 ? max_train_sample_count_ : 1;
uint32_t max_train_sample_count = train_sample_ratio * holder->count();
max_train_sample_count = max_train_sample_count > max_train_sample_count_
? max_train_sample_count_
: max_train_sample_count;
std::vector<std::vector<uint8_t>> sample_vecs;
// Use a fixed seed for deterministic sampling across runs.
uint32_t x = 456321;
std::mt19937 gen(x);
std::uniform_real_distribution<float> dist(0, 1);
uint32_t vec_size = meta.element_size();
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator for holder failed");
return IndexError_Runtime;
}
size_t sample_count = 0;
while (iter->is_valid() && sample_count < max_train_sample_count) {
float random = dist(gen);
if (random < train_sample_ratio) {
const void *vec = iter->data();
std::vector<uint8_t> temp_vec;
temp_vec.resize(vec_size);
std::memcpy(reinterpret_cast<uint8_t *>(&temp_vec[0]), vec, vec_size);
sample_vecs.push_back(std::move(temp_vec));
sample_count++;
}
iter->next();
}
sample_size = sample_vecs.size();
sample_data.reserve(sample_size * vec_size);
for (size_t i = 0; i < sample_size; i++) {
sample_data.append(reinterpret_cast<const char *>(sample_vecs[i].data()),
vec_size);
}
return 0;
}
template <typename T>
int DiskAnnPqTrainer::prepare_pq_train_data(
const IndexMeta &meta, size_t num_train, std::string &train_data,
bool use_zero_mean, std::vector<uint8_t> &centroid,
std::shared_ptr<CompactIndexFeatures> &train_features) {
uint32_t dim = meta.dimension();
uint32_t vec_size = meta.element_size();
std::string train_data_processed;
train_data_processed.resize(num_train * vec_size);
std::memcpy(&(train_data_processed[0]), train_data.data(),
num_train * vec_size);
// use fp32 to accumulate to avoid overflow
std::vector<float> centroid_temp(dim);
for (uint64_t d = 0; d < dim; d++) {
centroid_temp[d] = 0;
}
T *train_data_processed_ptr = reinterpret_cast<T *>(&train_data_processed[0]);
if (use_zero_mean) {
for (uint64_t d = 0; d < dim; d++) {
for (uint64_t p = 0; p < num_train; p++) {
centroid_temp[d] += train_data_processed_ptr[p * dim + d];
}
centroid_temp[d] /= num_train;
}
for (uint64_t d = 0; d < dim; d++) {
for (uint64_t p = 0; p < num_train; p++) {
train_data_processed_ptr[p * dim + d] -= centroid_temp[d];
}
}
}
for (size_t i = 0; i < num_train; ++i) {
train_features->emplace(train_data_processed_ptr + i * dim);
}
// copy the centroid out
centroid.resize(vec_size);
T *centroid_ptr = reinterpret_cast<T *>(centroid.data());
for (uint64_t d = 0; d < dim; d++) {
centroid_ptr[d] = centroid_temp[d];
}
return 0;
}
template <typename T>
int DiskAnnPqTrainer::convert_pivot_data(
const IndexMeta &meta, uint32_t num_centers, uint32_t pq_chunk_num,
const std::vector<uint32_t> &chunk_dims,
const std::vector<uint32_t> &chunk_offsets,
IndexCluster::CentroidList &centroids,
std::vector<uint8_t> &full_pivot_data) {
uint32_t dim = meta.dimension();
uint32_t element_size = meta.element_size();
full_pivot_data.resize(num_centers * element_size);
for (size_t chunk = 0; chunk < pq_chunk_num; ++chunk) {
for (size_t cluster = 0; cluster < num_centers; ++cluster) {
size_t idx = chunk * num_centers + cluster;
T *pivot_data_ptr = reinterpret_cast<T *>(&(full_pivot_data[0])) +
cluster * dim + chunk_offsets[chunk];
const T *feature_ptr =
reinterpret_cast<const T *>(centroids[idx].feature());
for (size_t d = 0; d <= chunk_dims[chunk]; ++d) {
pivot_data_ptr[d] = feature_ptr[d];
}
}
}
return 0;
}
int DiskAnnPqTrainer::train_pq(IndexThreads::Pointer threads,
const IndexMeta &meta, std::string &train_data,
size_t num_train, uint32_t num_centers,
uint32_t pq_chunk_num, uint32_t max_iterations,
bool use_zero_mean,
std::vector<uint8_t> &full_pivot_data,
std::vector<uint8_t> &centroid,
std::vector<uint32_t> &chunk_offsets) {
uint32_t dim = meta.dimension();
if (pq_chunk_num > dim) {
LOG_ERROR("Error: number of chunks more than dimension. chunk: %u, dim: %u",
pq_chunk_num, dim);
return IndexError_InvalidArgument;
}
std::shared_ptr<CompactIndexFeatures> train_features(
new CompactIndexFeatures(meta));
uint32_t type = meta.data_type();
int ret;
switch (type) {
case IndexMeta::DataType::DT_FP32:
ret = prepare_pq_train_data<float>(
meta, num_train, train_data, use_zero_mean, centroid, train_features);
if (ret != 0) {
LOG_ERROR("Failed to prepare pq train data");
return ret;
}
break;
case IndexMeta::DataType::DT_FP16:
ret = prepare_pq_train_data<ailego::Float16>(
meta, num_train, train_data, use_zero_mean, centroid, train_features);
if (ret != 0) {
LOG_ERROR("Failed to prepare pq train data");
return ret;
}
break;
}
// Do Train
ailego::Params params;
params.set(MULTI_CHUNK_CLUSTER_COUNT, num_centers);
params.set(MULTI_CHUNK_CLUSTER_CHUNK_COUNT, pq_chunk_num);
params.set(MULTI_CHUNK_CLUSTER_MAX_ITERATIONS, max_iterations);
ret = chunk_cluster_.init(meta, params);
if (ret != 0) {
LOG_ERROR("Failed to get chunk cluster");
return IndexError_InvalidArgument;
}
ret = chunk_cluster_.mount(train_features);
if (ret != 0) {
LOG_ERROR("Cannot mount train features");
return ret;
}
std::vector<uint32_t> labels;
ret = chunk_cluster_.cluster(threads, cluster_centroids_);
if (ret != 0) {
LOG_ERROR("Failed to cluster");
return ret;
}
chunk_offsets = chunk_cluster_.chunk_dim_offsets();
auto chunk_dims = chunk_cluster_.chunk_dims();
switch (type) {
case IndexMeta::DataType::DT_FP32:
ret = convert_pivot_data<float>(meta, num_centers, pq_chunk_num,
chunk_dims, chunk_offsets,
cluster_centroids_, full_pivot_data);
if (ret != 0) {
LOG_ERROR("Failed to convert pivot data");
return ret;
}
break;
case IndexMeta::DataType::DT_FP16:
ret = convert_pivot_data<ailego::Float16>(
meta, num_centers, pq_chunk_num, chunk_dims, chunk_offsets,
cluster_centroids_, full_pivot_data);
if (ret != 0) {
LOG_ERROR("Failed to convert pivot data");
return ret;
}
break;
}
return 0;
}
int DiskAnnPqTrainer::train_quantized_data(
IndexThreads::Pointer threads, IndexHolder::Pointer holder,
const IndexMeta &meta, std::vector<uint8_t> &pq_full_pivot_data,
std::vector<uint8_t> &pq_centroid, std::vector<uint32_t> &pq_chunk_offsets,
size_t pq_chunk_num) {
size_t train_size;
std::string train_data;
int ret = gen_random_sample(holder, meta, train_data, train_size);
if (ret != 0) {
LOG_ERROR("Get Random Sample Error, ret: %d", ret);
return ret;
}
LOG_INFO("Training data with %zu samples loaded.", train_size);
// bool use_zero_mean = (meta.metric_name() != "InnerProduct" ? true :
// false);
bool use_zero_mean = false;
ret = train_pq(threads, meta, train_data, train_size, PQTable::kPQCentroidNum,
pq_chunk_num, PQTable::kMeanIterNum, use_zero_mean,
pq_full_pivot_data, pq_centroid, pq_chunk_offsets);
if (ret != 0) {
LOG_ERROR("Train PQ Error, ret: %d", ret);
return ret;
}
return 0;
}
int DiskAnnPqTrainer::generate_pq(IndexThreads::Pointer threads,
const IndexMeta &meta,
IndexHolder::Pointer holder,
uint32_t pq_chunk_num,
std::vector<uint8_t> &centroid,
std::vector<uint8_t> &block_compressed_data) {
uint32_t type = meta.data_type();
uint32_t dim = meta.dimension();
if (pq_chunk_num > dim) {
LOG_ERROR("Error: number of chunks more than dimension. chunk: %u, dim: %u",
pq_chunk_num, dim);
return IndexError_InvalidArgument;
}
// Do Label
std::vector<uint32_t> labels;
size_t num_vecs = holder->count();
size_t batch_size =
num_vecs <= compress_batch_size_ ? num_vecs : compress_batch_size_;
std::vector<uint32_t> block_compressed_base(batch_size * pq_chunk_num);
std::memset(&block_compressed_base[0], 0,
batch_size * pq_chunk_num * sizeof(uint32_t));
std::vector<uint8_t> block_data(batch_size * meta.element_size());
std::vector<uint8_t> block_data_converted(batch_size * meta.element_size());
size_t block_num = DiskAnnUtil::div_round_up(num_vecs, batch_size);
block_compressed_data.resize(num_vecs * pq_chunk_num);
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator for holder failed");
return IndexError_Runtime;
}
for (size_t block = 0; block < block_num; block++) {
size_t start_id = block * batch_size;
size_t end_id = std::min((block + 1) * batch_size, num_vecs);
size_t cur_block_size = end_id - start_id;
for (size_t i = 0; i < cur_block_size && iter->is_valid(); i++) {
const void *vec = iter->data();
std::memcpy(
reinterpret_cast<uint8_t *>(&block_data[0]) + i * meta.element_size(),
vec, meta.element_size());
iter->next();
}
std::memcpy(block_data_converted.data(), block_data.data(),
cur_block_size * meta.element_size());
LOG_INFO("Processing Docs, Range: [%zu, %zu)..", start_id, end_id);
std::shared_ptr<CompactIndexFeatures> block_features(
new CompactIndexFeatures(meta));
switch (type) {
case IndexMeta::DataType::DT_FP32:
DiskAnnUtil::convert_vector_to_residual<float>(
reinterpret_cast<float *>(block_data_converted.data()),
cur_block_size, dim, centroid.data());
break;
case IndexMeta::DataType::DT_FP16:
DiskAnnUtil::convert_vector_to_residual<ailego::Float16>(
reinterpret_cast<ailego::Float16 *>(block_data_converted.data()),
cur_block_size, dim, centroid.data());
break;
default:
return IndexError_InvalidArgument;
}
for (size_t i = 0; i < cur_block_size; i++) {
block_features->emplace(block_data_converted.data() +
i * meta.element_size());
}
int ret = chunk_cluster_.mount(block_features);
if (ret != 0) {
LOG_ERROR("Cannot mount block features");
return ret;
}
ret = chunk_cluster_.label(threads, cluster_centroids_, &labels);
if (ret != 0) {
LOG_ERROR("Failed to label");
return ret;
}
std::vector<uint8_t> compressed_data(cur_block_size * pq_chunk_num);
DiskAnnUtil::convert_types_uint32_to_uint8(
labels.data(), compressed_data.data(), cur_block_size, pq_chunk_num);
memcpy(&(block_compressed_data[0]) + start_id * pq_chunk_num,
compressed_data.data(), cur_block_size * pq_chunk_num);
LOG_INFO("Generate PQ Data Done.");
}
return 0;
}
int DiskAnnPqTrainer::generate_quantized_data(
IndexThreads::Pointer threads, IndexHolder::Pointer holder,
const IndexMeta &meta, std::vector<uint8_t> &pq_centroid,
std::vector<uint8_t> &block_compressed_data, size_t pq_chunk_num) {
// bool use_zero_mean = (meta.metric_name() != "InnerProduct" ? true :
// false);
int ret = generate_pq(threads, meta, holder, pq_chunk_num, pq_centroid,
block_compressed_data);
if (ret != 0) {
LOG_ERROR("Generate PQ Error, ret: %d", ret);
return ret;
}
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,87 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/core/framework/index_framework.h>
#include "diskann_entity.h"
#include "diskann_pq_table.h"
#include "../cluster/multi_chunk_cluster.h"
namespace zvec {
namespace core {
class DiskAnnPqTrainer {
public:
typedef std::unique_ptr<DiskAnnPqTrainer> UPointer;
public:
DiskAnnPqTrainer(uint32_t max_train_sample_count);
virtual ~DiskAnnPqTrainer();
public:
template <typename T>
static int prepare_pq_train_data(
const IndexMeta &meta, size_t num_train, std::string &train_data,
bool use_zero_mean, std::vector<uint8_t> &centroid,
std::shared_ptr<CompactIndexFeatures> &train_features);
template <typename T>
static int convert_pivot_data(const IndexMeta &meta, uint32_t num_centers,
uint32_t pq_chunk_num,
const std::vector<uint32_t> &chunk_dims,
const std::vector<uint32_t> &chunk_offsets,
IndexCluster::CentroidList &centroids,
std::vector<uint8_t> &full_pivot_data);
int gen_random_sample(IndexHolder::Pointer holder, const IndexMeta &meta,
std::string &sample_data, size_t &sample_size);
int generate_quantized_data(IndexThreads::Pointer threads,
IndexHolder::Pointer holder,
const IndexMeta &meta,
std::vector<uint8_t> &pq_centroid,
std::vector<uint8_t> &block_compressed_data,
size_t num_pq_chunks);
int generate_pq(IndexThreads::Pointer threads, const IndexMeta &meta,
IndexHolder::Pointer holder, uint32_t num_pq_chunks,
std::vector<uint8_t> &centroid,
std::vector<uint8_t> &block_compressed_data);
int train_quantized_data(IndexThreads::Pointer threads,
IndexHolder::Pointer holder, const IndexMeta &meta,
std::vector<uint8_t> &pq_full_pivot_data,
std::vector<uint8_t> &pq_centroid,
std::vector<uint32_t> &pq_chunk_offsets,
size_t num_pq_chunks);
int train_pq(IndexThreads::Pointer threads, const IndexMeta &meta,
std::string &train_data, size_t num_train, uint32_t num_centers,
uint32_t num_pq_chunks, uint32_t max_iterations,
bool use_zero_mean, std::vector<uint8_t> &full_pivot_data,
std::vector<uint8_t> &centroid,
std::vector<uint32_t> &chunk_offsets);
private:
static constexpr uint32_t compress_batch_size_{
DiskAnnEntity::kDefaultCompressBatchSize};
// pq cluster
MultiChunkCluster chunk_cluster_;
IndexCluster::CentroidList cluster_centroids_;
uint32_t max_train_sample_count_{PQTable::kMaxTrainSampleCount};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,190 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_reducer.h"
#include <vector>
#include <ailego/parallel/lock.h>
#include <ailego/parallel/multi_thread_list.h>
#include <zvec/ailego/utility/string_helper.h>
#include <zvec/core/framework/index_reducer.h>
#include "diskann_params.h"
namespace zvec {
namespace core {
int DiskAnnReducer::init(const ailego::Params &params) {
params.get(PARAM_DISKANN_REDUCER_WORKING_PATH, &working_path_);
if (working_path_.empty()) {
LOG_ERROR("Missing parameter. %s",
PARAM_DISKANN_REDUCER_WORKING_PATH.c_str());
return IndexError_InvalidArgument;
}
std::string index_name =
params.get_as_string(PARAM_DISKANN_REDUCER_INDEX_NAME);
if (index_name.empty()) {
index_name = std::to_string(std::clock());
}
reducer_file_path_ = ailego::StringHelper::Concat(
working_path_, "/", kReducerFileName, index_name);
holder_file_path_ = ailego::StringHelper::Concat(working_path_, "/",
kHolderFileName, index_name);
state_ = STATE_INITED;
return 0;
}
int DiskAnnReducer::cleanup(void) {
return 0;
}
//! Reduce operator with filter
int DiskAnnReducer::reduce(const IndexFilter &filter) {
if (entities_.empty() || state_ != STATE_FEED) {
LOG_ERROR("No container to reduce, feed first");
return IndexError_NoReady;
}
if (use_mem_holder_) {
mem_holder_ = std::make_shared<RandomAccessIndexHolder>(meta_);
for (auto entity : entities_) {
size_t doc_cnt = entity->doc_cnt();
for (size_t id = 0; id < doc_cnt; ++id) {
diskann_key_t pkey = entity->get_key(id);
if (filter.is_valid() && filter(pkey)) {
continue;
}
const void *vec = entity->get_vector(id);
mem_holder_->emplace(pkey, vec);
}
}
} else {
disk_holder_ =
std::make_shared<DiskAnnIndexHolder>(meta_, holder_file_path_);
int ret = disk_holder_->init();
if (ret != 0) {
LOG_ERROR("DiskAnn Index Holder init failed");
return ret;
}
for (auto entity : entities_) {
size_t doc_cnt = entity->doc_cnt();
for (size_t id = 0; id < doc_cnt; ++id) {
diskann_key_t pkey = entity->get_key(id);
if (filter.is_valid() && filter(pkey)) {
continue;
}
const void *vec = entity->get_vector(id);
disk_holder_->emplace(pkey, vec);
}
}
disk_holder_->close();
}
builder_ = IndexFactory::CreateBuilder(kDiskAnnBuilderName);
if (!builder_) {
LOG_ERROR("Create builder failed. name[%s]", kDiskAnnBuilderName.c_str());
return IndexError_Runtime;
}
if (thread_pool_ == nullptr) {
LOG_ERROR(
"Only support multi-thread mode. Thread pool is not set for reducer.");
return IndexError_NoReady;
}
LOG_INFO("Start diskann reduce");
ailego::ElapsedTime timer;
auto params = meta_.builder_params();
int ret = builder_->init(meta_, params);
if (ret != 0) {
LOG_ERROR("Init proxima streamer failed. ret[%d]", ret);
return ret;
}
if (use_mem_holder_) {
ret = builder_->train(mem_holder_);
if (ret != 0) {
LOG_ERROR("Diskann builder failed to train. ret[%d]", ret);
return ret;
}
ret = builder_->build(mem_holder_);
if (ret != 0) {
LOG_ERROR("Diskann builder failed to build. ret[%d]", ret);
return ret;
}
} else {
ret = builder_->train(disk_holder_);
if (ret != 0) {
LOG_ERROR("Diskann builder failed to train. ret[%d]", ret);
return ret;
}
ret = builder_->build(disk_holder_);
if (ret != 0) {
LOG_ERROR("Diskann builder failed to build. ret[%d]", ret);
return ret;
}
}
auto &stats = builder_->stats();
stats_.set_reduced_costtime(timer.seconds());
stats_.set_filtered_count(stats.discarded_count());
state_ = STATE_REDUCE;
LOG_INFO("End DiskAnn reduce. cost time: [%zu]s", (size_t)timer.seconds());
return 0;
}
//! Dump index by dumper
int DiskAnnReducer::dump(const IndexDumper::Pointer &dumper) {
LOG_INFO("Begin diskann reducer dump");
if (state_ != STATE_REDUCE) {
LOG_WARN("Reduce first before dump.");
return IndexError_NoReady;
}
ailego::ElapsedTime timer;
int ret = builder_->dump(dumper);
if (ret != 0) {
LOG_ERROR("diskann reducer dump failed. ret[%d]", ret);
return ret;
}
stats_.set_dumped_costtime(timer.seconds());
LOG_INFO("End diskann reducer dump, dump costtime=[%zu]s",
(size_t)(stats_.dumped_costtime()));
return 0;
}
INDEX_FACTORY_REGISTER_REDUCER(DiskAnnReducer);
} // namespace core
} // namespace zvec
@@ -0,0 +1,85 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <vector>
#include <ailego/parallel/lock.h>
#include <ailego/parallel/multi_thread_list.h>
#include <utility/sparse_utility.h>
#include <zvec/core/framework/index_holder.h>
#include <zvec/core/framework/index_reducer.h>
#include "diskann_holder.h"
#include "diskann_reducer_entity.h"
namespace zvec {
namespace core {
class DiskAnnReducer : public IndexReducer {
public:
//! Constructor
DiskAnnReducer(void) = default;
protected:
//! Initialize Reducer
virtual int init(const ailego::Params &params) override;
//! Cleanup Reducer
virtual int cleanup(void) override;
//! Feed indexes from containers
// virtual int feed(IndexStorage::Pointer container) override;
//! Reduce operator (with filter)
virtual int reduce(const IndexFilter &filter) override;
//! Dump index by dumper
virtual int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
return stats_;
}
private:
enum State {
STATE_UNINITED = 0,
STATE_INITED = 1,
STATE_FEED = 2,
STATE_REDUCE = 3
};
std::string working_path_{""};
IndexMeta meta_{};
std::vector<DiskAnnReducerEntity::Pointer> entities_{};
// bool use_mem_holder_{true};
bool use_mem_holder_{false};
RandomAccessIndexHolder::Pointer mem_holder_;
DiskAnnIndexHolder::Pointer disk_holder_;
IndexBuilder::Pointer builder_{nullptr};
std::string reducer_file_path_{""};
std::string holder_file_path_{""};
Stats stats_{};
State state_{STATE_UNINITED};
const std::string kDiskAnnBuilderName{"DiskAnnBuilder"};
const std::string kReducerFileName{"diskann.reducer.builder."};
const std::string kHolderFileName{"diskann.reducer.holder."};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,215 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_reducer_entity.h"
#include <zvec/ailego/parallel/thread_pool.h>
#include <zvec/core/framework/index_holder.h>
namespace zvec {
namespace core {
int DiskAnnReducerEntity::load(const IndexStorage::Pointer &container,
bool check_crc) {
container_ = container;
int ret = load_segments(check_crc);
if (ret != 0) {
return ret;
}
sector_num_per_node_ = node_per_sector() > 0
? 1
: DiskAnnUtil::div_round_up(
max_node_size(), DiskAnnUtil::kSectorSize);
loaded_ = true;
return 0;
}
int DiskAnnReducerEntity::load_segments(bool /*check_crc*/) {
int ret;
ret = load_header_segment();
if (ret != 0) {
LOG_ERROR("Load Header Segment Failed, ret = %d", ret);
return ret;
}
ret = load_key_segment();
if (ret != 0) {
LOG_ERROR("Load Key Segment Failed, ret = %d", ret);
return ret;
}
ret = load_vector_segment();
if (ret != 0) {
LOG_ERROR("Load Vector Segment Failed, ret = %d", ret);
return ret;
}
return 0;
}
int DiskAnnReducerEntity::load_header_segment() {
const void *data = nullptr;
meta_segment_ = container_->get(kDiskAnnMetaSegmentId);
if (!meta_segment_ ||
meta_segment_->data_size() < sizeof(DiskAnnMetaHeader)) {
LOG_ERROR("Miss or invalid segment %s", kDiskAnnMetaSegmentId.c_str());
return IndexError_InvalidFormat;
}
if (meta_segment_->read(0, reinterpret_cast<const void **>(&data),
sizeof(DiskAnnMetaHeader)) !=
sizeof(DiskAnnMetaHeader)) {
LOG_ERROR("Read segment %s failed", kDiskAnnMetaSegmentId.c_str());
return IndexError_ReadData;
}
::memcpy(reinterpret_cast<uint8_t *>(&meta_header_), data,
sizeof(DiskAnnMetaHeader));
return 0;
}
int DiskAnnReducerEntity::load_vector_segment() {
vector_segment_ = container_->get(kDiskAnnVectorSegmentId);
if (!vector_segment_) {
LOG_ERROR("Miss or invalid segment %s",
DiskAnnEntity::kDiskAnnVectorSegmentId.c_str());
return IndexError_InvalidFormat;
}
return 0;
}
int DiskAnnReducerEntity::load_key_segment() {
// load key
key_segment_ = container_->get(kDiskAnnKeySegmentId);
if (!key_segment_) {
LOG_ERROR("Miss or invalid segment %s",
DiskAnnEntity::kDiskAnnKeySegmentId.c_str());
return IndexError_InvalidFormat;
}
size_t key_data_len = doc_cnt() * sizeof(key_t);
// load key mapping
key_mapping_segment_ = container_->get(kDiskAnnKeyMappingSegmentId);
const void *data = nullptr;
if (key_mapping_segment_->read(0, reinterpret_cast<const void **>(&data),
key_data_len) != key_data_len) {
LOG_ERROR("Read segment %s failed", kDiskAnnKeyMappingSegmentId.c_str());
return IndexError_ReadData;
}
key_buffer_.resize(key_data_len);
memcpy(&(key_buffer_[0]), data, key_data_len);
return 0;
}
bool DiskAnnReducerEntity::do_crc_check(
std::vector<SegmentPointer> &segments) const {
constexpr size_t blk_size = 4096;
const void *data;
for (auto &segment : segments) {
size_t offset = 0;
size_t rd_size;
uint32_t crc = 0;
while (offset < segment->data_size()) {
size_t size = std::min(blk_size, segment->data_size() - offset);
if ((rd_size = segment->read(offset, &data, size)) <= 0) {
break;
}
offset += rd_size;
crc = ailego::Crc32c::Hash(data, rd_size, crc);
}
if (crc != segment->data_crc()) {
return false;
}
}
return true;
}
//! Get vector local id by key
diskann_id_t DiskAnnReducerEntity::get_id(diskann_key_t key) const {
const diskann_id_t *key_mapping_data_ptr =
reinterpret_cast<const diskann_id_t *>(key_mapping_buffer_.data());
const diskann_key_t *key_data_ptr =
reinterpret_cast<const diskann_key_t *>(key_buffer_.data());
//! Do binary search
diskann_id_t start = 0UL;
diskann_id_t end = doc_cnt();
diskann_id_t idx = 0u;
while (start < end) {
idx = start + (end - start) / 2;
diskann_id_t local_id = key_mapping_data_ptr[idx];
const diskann_key_t local_key = key_data_ptr[local_id];
if (local_key < key) {
start = idx + 1;
} else if (local_key > key) {
end = idx;
} else {
return local_id;
}
}
return kInvalidId;
}
diskann_key_t DiskAnnReducerEntity::get_key(diskann_id_t id) const {
const void *key;
if (ailego_unlikely(key_segment_->read(id * sizeof(diskann_key_t), &key,
sizeof(diskann_key_t)) !=
sizeof(diskann_key_t))) {
LOG_ERROR("Read key from segment failed");
return kInvalidKey;
}
return *(reinterpret_cast<const diskann_key_t *>(key));
}
const void *DiskAnnReducerEntity::get_vector(diskann_id_t id) const {
size_t read_size = sector_num_per_node_ * DiskAnnUtil::kSectorSize;
size_t sector_id = DiskAnnUtil::get_node_sector(
node_per_sector(), max_node_size(), DiskAnnUtil::kSectorSize, id);
size_t offset = sector_id * DiskAnnUtil::kSectorSize;
if (sector_id != sector_id_) {
const void *sector_data;
if (ailego_unlikely(vector_segment_->read(offset, &sector_data,
read_size) != read_size)) {
LOG_ERROR("Read vector from segment failed");
return nullptr;
}
sector_id_ = sector_id;
sector_buffer_.assign(reinterpret_cast<const char *>(sector_data),
read_size);
}
return DiskAnnUtil::offset_to_node_const(
node_per_sector(), max_node_size(),
reinterpret_cast<const uint8_t *>(sector_buffer_.data()), id);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,69 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <mutex>
#include <set>
#include <zvec/ailego/parallel/thread_pool.h>
#include <zvec/core/framework/index_holder.h>
#include "diskann_entity.h"
#include "diskann_file_reader.h"
#include "diskann_pq_table.h"
#include "diskann_util.h"
namespace zvec {
namespace core {
class DiskAnnReducerEntity : public DiskAnnEntity {
public:
using Pointer = std::shared_ptr<DiskAnnReducerEntity>;
using SegmentPointer = IndexStorage::Segment::Pointer;
public:
DiskAnnReducerEntity() = default;
virtual ~DiskAnnReducerEntity() = default;
int load(const IndexStorage::Pointer &container, bool check_crc);
int load_segments(bool check_crc);
int load_header_segment();
int load_vector_segment();
int load_key_segment();
int load_key_mapping_segment();
bool do_crc_check(std::vector<SegmentPointer> &segments) const;
diskann_id_t get_id(diskann_key_t key) const override;
diskann_key_t get_key(diskann_id_t id) const override;
const void *get_vector(diskann_id_t id) const override;
private:
IndexStorage::Pointer container_{};
IndexStorage::Segment::Pointer meta_segment_{};
IndexStorage::Segment::Pointer vector_segment_{};
IndexStorage::Segment::Pointer key_segment_{};
IndexStorage::Segment::Pointer key_mapping_segment_{};
std::string key_buffer_;
std::string key_mapping_buffer_;
size_t sector_num_per_node_{0};
mutable size_t sector_id_{-1U};
mutable std::string sector_buffer_;
bool loaded_{false};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,312 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_searcher.h"
#include "diskann_context.h"
#include "diskann_indexer.h"
#include "diskann_params.h"
namespace zvec {
namespace core {
DiskAnnSearcher::DiskAnnSearcher() {}
DiskAnnSearcher::~DiskAnnSearcher() {}
int DiskAnnSearcher::init(const ailego::Params &search_params) {
search_params.get(PARAM_DISKANN_SEARCHER_LIST_SIZE, &list_size_);
search_params.get(PARAM_DISKANN_SEARCHER_CACHE_NODE_NUM, &cache_nodes_num_);
return 0;
}
void DiskAnnSearcher::print_debug_info() {}
int DiskAnnSearcher::cleanup() {
LOG_INFO("Begin DiskAnnSearcher:cleanup");
LOG_INFO("End DiskAnnSearcher:cleanup");
return 0;
}
int DiskAnnSearcher::load(IndexStorage::Pointer storage,
IndexMetric::Pointer measure) {
LOG_INFO("DiskAnnSearcher::load Begin");
auto start_time = ailego::Monotime::MilliSeconds();
int ret = IndexHelper::DeserializeFromStorage(storage.get(), &meta_);
if (ret != 0) {
LOG_ERROR("Failed to deserialize meta from storage");
return ret;
}
ret = entity_.load(meta_, storage);
if (ret != 0) {
LOG_INFO("Searcher Entity Load Failed");
return ret;
}
diskann_indexer_ = std::make_shared<DiskAnnIndexer>(meta_);
int res = diskann_indexer_->init(entity_);
if (res != 0) {
return res;
}
if (cache_nodes_num_ != 0) {
std::vector<diskann_id_t> node_list;
LOG_INFO("Caching %u nodes around medoid(s)", cache_nodes_num_);
diskann_indexer_->cache_bfs_levels(cache_nodes_num_, node_list);
diskann_indexer_->load_cache_list(node_list);
node_list.clear();
node_list.shrink_to_fit();
}
if (measure) {
measure_ = measure;
} else {
measure_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!measure_) {
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
return IndexError_NoExist;
}
ret = measure_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
return ret;
}
if (measure_->query_metric()) {
measure_ = measure_->query_metric();
}
}
stats_.set_loaded_costtime(ailego::Monotime::MilliSeconds() - start_time);
state_ = STATE_LOADED;
magic_ = IndexContext::GenerateMagic();
LOG_INFO("DiskAnnSearcher::load Done");
return 0;
}
int DiskAnnSearcher::unload() {
LOG_INFO("DiskAnnSearcher unload index");
state_ = STATE_INITED;
return 0;
}
int DiskAnnSearcher::update_context(DiskAnnContext *ctx) const {
const DiskAnnEntity::Pointer entity = entity_.clone();
if (!entity) {
LOG_ERROR("Failed to clone search context entity");
return IndexError_Runtime;
}
return ctx->update_context(DiskAnnContext::kSearcherContext, meta_, measure_,
entity, magic_);
}
int DiskAnnSearcher::search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const {
// do search
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to DiskAnnContext failed");
return IndexError_Cast;
}
// Context is pooled per index type. When switching between DiskAnn indexes
// with different element sizes (e.g., fp16 vs fp32), the cached context has
// undersized buffers. Recreate it to ensure correct buffer allocations.
if (ctx->magic() != magic_) {
uint32_t saved_topk = ctx->topk();
context = create_context();
if (!context) {
LOG_ERROR("Failed to recreate context for current streamer");
return IndexError_Runtime;
}
ctx = dynamic_cast<DiskAnnContext *>(context.get());
ctx->set_topk(saved_topk);
}
ctx->clear();
ctx->resize_results(count);
for (uint32_t i = 0; i < count; i++) {
ctx->reset_query(query);
diskann_indexer_->knn_search(ctx);
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
ctx->topk_to_result(i);
query = static_cast<const char *>(query) + qmeta.element_size();
}
return 0;
}
int DiskAnnSearcher::search_bf_impl(const void *query,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to DiskAnnContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer, recreate it
//! to ensure buffers are correctly sized for this index's parameters.
uint32_t saved_topk = ctx->topk();
context = create_context();
if (!context) {
LOG_ERROR("Failed to recreate context for current streamer");
return IndexError_Runtime;
}
ctx = dynamic_cast<DiskAnnContext *>(context.get());
ctx->set_topk(saved_topk);
}
ctx->clear();
ctx->resize_results(count);
for (size_t i = 0; i < count; ++i) {
ctx->reset_query(query);
diskann_indexer_->linear_search(ctx);
ctx->topk_to_result(i);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int DiskAnnSearcher::search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to DiskAnnContext failed");
return IndexError_Cast;
}
if (ailego_unlikely(p_keys.size() != count)) {
LOG_ERROR("The size of p_keys is not equal to count");
return IndexError_InvalidArgument;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer, recreate it
//! to ensure buffers are correctly sized for this index's parameters.
uint32_t saved_topk = ctx->topk();
context = create_context();
if (!context) {
LOG_ERROR("Failed to recreate context for current streamer");
return IndexError_Runtime;
}
ctx = dynamic_cast<DiskAnnContext *>(context.get());
ctx->set_topk(saved_topk);
}
ctx->clear();
ctx->resize_results(count);
for (size_t i = 0; i < count; ++i) {
ctx->reset_query(query);
diskann_indexer_->keys_search(p_keys[i], ctx);
ctx->topk_to_result(i);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int DiskAnnSearcher::get_vector(uint64_t key, Context::Pointer &context,
std::string &vector) const {
return diskann_indexer_->get_vector(key, context, vector);
}
IndexSearcher::Context::Pointer DiskAnnSearcher::create_context() const {
const DiskAnnEntity::Pointer search_ctx_entity = entity_.clone();
if (!search_ctx_entity) {
LOG_ERROR("Failed to create search context entity");
return Context::Pointer();
}
DiskAnnContext *ctx =
new (std::nothrow) DiskAnnContext(meta_, measure_, search_ctx_entity);
if (ctx == nullptr) {
LOG_ERROR("Failed to allocate DiskAnn Context");
return Context::Pointer();
}
if (ailego_unlikely(ctx->init(
DiskAnnContext::kSearcherContext, search_ctx_entity->max_degree(),
search_ctx_entity->pq_chunk_num(), meta_.element_size())) != 0) {
LOG_ERROR("Init DiskAnn Context failed");
delete ctx;
return Context::Pointer();
}
ctx->set_list_size(list_size_);
ctx->set_magic(magic_);
return Context::Pointer(ctx);
}
INDEX_FACTORY_REGISTER_SEARCHER(DiskAnnSearcher);
} // namespace core
} // namespace zvec
@@ -0,0 +1,166 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/core/framework/index_framework.h>
#include "diskann_context.h"
#include "diskann_indexer.h"
class LinuxAlignedFileReader;
namespace zvec {
namespace core {
class DiskAnnSearcher : public IndexSearcher {
public:
using ContextPointer = IndexSearcher::Context::Pointer;
public:
DiskAnnSearcher(void);
~DiskAnnSearcher(void);
DiskAnnSearcher(const DiskAnnSearcher &) = delete;
DiskAnnSearcher &operator=(const DiskAnnSearcher &) = delete;
protected:
//! Initialize Searcher
int init(const ailego::Params &params) override;
//! Cleanup Searcher
int cleanup(void) override;
//! Load Index from storage
int load(IndexStorage::Pointer storage, IndexMetric::Pointer metric) override;
//! Unload index from storage
int unload(void) override;
//! KNN Search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_impl(query, qmeta, 1, context);
}
//! KNN Search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, ContextPointer &context) const override;
//! Linear Search
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_impl(query, qmeta, 1, context);
}
//! Linear Search
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, ContextPointer &context) const override;
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query, const uint32_t sparse_count,
const uint32_t *sparse_indices,
const void *sparse_query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, &sparse_count, sparse_indices,
sparse_query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
int search_bf_by_p_keys_impl(
const void * /*query*/, const uint32_t * /*sparse_count*/,
const uint32_t * /*sparse_indices*/, const void * /*sparse_query*/,
const std::vector<std::vector<uint64_t>> & /*p_keys*/,
const IndexQueryMeta & /*qmeta*/, uint32_t /*count*/,
ContextPointer & /*context*/) const override {
return IndexError_NotImplemented;
}
//! Get vector by key
int get_vector(uint64_t key, Context::Pointer &context,
std::string &vector) const override;
//! Create a searcher context
ContextPointer create_context() const override;
//! Create a new iterator
IndexSearcher::Provider::Pointer create_provider(void) const override {
return nullptr;
}
//! Retrieve statistics
const Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
const IndexMeta &meta(void) const override {
return meta_;
}
//! Retrieve params of index
const ailego::Params &params(void) const override {
return params_;
}
void print_debug_info() override;
private:
template <typename T, typename LabelT = uint32_t>
int search_disk_index(const std::string &query_file,
const uint32_t num_nodes_to_cache,
const uint32_t recall_at, const uint32_t beamwidth);
//! To share ctx across streamer/searcher, we need to update the context for
//! current streamer/searcher
int update_context(DiskAnnContext *ctx) const;
private:
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
IndexMetric::Pointer measure_{};
IndexMeta meta_{};
ailego::Params params_{};
uint32_t list_size_{200};
uint32_t cache_nodes_num_{0};
bool warm_up_{false};
uint32_t beam_size_{2};
DiskAnnIndexer::Pointer diskann_indexer_{nullptr};
DiskAnnSearcherEntity entity_{};
uint32_t magic_{0U};
Stats stats_;
State state_{STATE_INIT};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,441 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_searcher_entity.h"
namespace zvec {
namespace core {
const DiskAnnEntity::Pointer DiskAnnSearcherEntity::clone() const {
auto meta_segment = meta_segment_->clone();
if (ailego_unlikely(!meta_segment)) {
LOG_ERROR("clone segment %s failed", kDiskAnnMetaSegmentId.c_str());
return DiskAnnEntity::Pointer();
}
auto pq_meta_segment = pq_meta_segment_->clone();
if (ailego_unlikely(!pq_meta_segment)) {
LOG_ERROR("clone segment %s failed", kDiskAnnPqMetaSegmentId.c_str());
return DiskAnnEntity::Pointer();
}
auto pq_data_segment = pq_data_segment_->clone();
if (ailego_unlikely(!pq_data_segment)) {
LOG_ERROR("clone segment %s failed", kDiskAnnPqDataSegmentId.c_str());
return DiskAnnEntity::Pointer();
}
auto vector_segment = vector_segment_->clone();
if (ailego_unlikely(!vector_segment)) {
LOG_ERROR("clone segment %s failed", kDiskAnnVectorSegmentId.c_str());
return DiskAnnEntity::Pointer();
}
auto key_segment = key_segment_->clone();
if (ailego_unlikely(!key_segment)) {
LOG_ERROR("clone segment %s failed", kDiskAnnKeySegmentId.c_str());
return DiskAnnEntity::Pointer();
}
auto key_mapping_segment = key_mapping_segment_->clone();
if (ailego_unlikely(!key_mapping_segment)) {
LOG_ERROR("clone segment %s failed", kDiskAnnKeyMappingSegmentId.c_str());
return DiskAnnEntity::Pointer();
}
auto entrypoint_segment = entrypoint_segment_->clone();
if (ailego_unlikely(!entrypoint_segment)) {
LOG_ERROR("clone segment %s failed", kDiskAnnEntryPointSegmentId.c_str());
return DiskAnnEntity::Pointer();
}
DiskAnnSearcherEntity *entity = new (std::nothrow) DiskAnnSearcherEntity(
meta_header_, pq_meta_, meta_segment, pq_meta_segment, pq_data_segment,
vector_segment, key_segment, key_mapping_segment, entrypoint_segment,
num_threads_, list_size_, cache_nodes_num_, warm_up_, beam_size_, meta_,
pq_table_, key_buffer_, key_mapping_buffer_, entrypoints_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("DiskAnnSearcherEntity new failed");
}
return DiskAnnEntity::Pointer(entity);
}
int DiskAnnSearcherEntity::load(const IndexMeta &meta,
IndexStorage::Pointer storage) {
meta_ = meta;
storage_ = storage;
int ret;
ret = load_header_segment();
if (ret != 0) {
LOG_ERROR("Load Header Segment Failed, ret = %d", ret);
return ret;
}
ret = load_pq_segment();
if (ret != 0) {
LOG_ERROR("Load PQ Meta Segment Failed, ret = %d", ret);
return ret;
}
ret = load_key_segment();
if (ret != 0) {
LOG_ERROR("Load Key Segment Failed, ret = %d", ret);
return ret;
}
ret = load_key_mapping_segment();
if (ret != 0) {
LOG_ERROR("Load Key Segment Failed, ret = %d", ret);
return ret;
}
ret = load_entrypoint_segment();
if (ret != 0) {
LOG_WARN("Load EntryPoint Segment Failed, ret = %d", ret);
return ret;
}
ret = load_vector_segment();
if (ret != 0) {
LOG_ERROR("Load Vector Segment Failed, ret = %d", ret);
return ret;
}
return 0;
}
int DiskAnnSearcherEntity::load_pq_segment() {
const void *data = nullptr;
// load pq meta
pq_meta_segment_ = storage_->get(DiskAnnEntity::kDiskAnnPqMetaSegmentId);
if (!pq_meta_segment_) {
LOG_ERROR("Miss or invalid segment %s",
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str());
return IndexError_InvalidFormat;
}
size_t read_size;
size_t offset = 0;
// 1. read pq meta
read_size = pq_meta_segment_->read(offset, &data, sizeof(DiskAnnPqMeta));
if (read_size != sizeof(DiskAnnPqMeta)) {
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
sizeof(DiskAnnPqMeta), read_size);
return IndexError_ReadData;
}
memcpy(reinterpret_cast<uint8_t *>(&pq_meta_), data, sizeof(DiskAnnPqMeta));
offset += read_size;
// 2. read full pivot data
std::vector<uint8_t> full_pivot_data;
full_pivot_data.resize(pq_meta_.full_pivot_data_size);
read_size =
pq_meta_segment_->read(offset, &data, pq_meta_.full_pivot_data_size);
if (read_size != pq_meta_.full_pivot_data_size) {
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
(size_t)(pq_meta_.full_pivot_data_size), (size_t)read_size);
return IndexError_ReadData;
}
memcpy(&(full_pivot_data[0]), data, read_size);
offset += read_size;
// 3. read centroid
std::vector<uint8_t> centroid;
centroid.resize(pq_meta_.centroid_data_size);
read_size =
pq_meta_segment_->read(offset, &data, pq_meta_.centroid_data_size);
if (read_size != pq_meta_.centroid_data_size) {
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
(size_t)(pq_meta_.centroid_data_size), (size_t)read_size);
return IndexError_ReadData;
}
memcpy(&(centroid[0]), data, read_size);
offset += read_size;
// 4. chunk offset
std::vector<uint32_t> chunk_offsets;
chunk_offsets.resize(pq_meta_.chunk_num + 1);
read_size = pq_meta_segment_->read(
offset, &data, (pq_meta_.chunk_num + 1) * sizeof(uint32_t));
if (read_size != (pq_meta_.chunk_num + 1) * sizeof(uint32_t)) {
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
(size_t)((pq_meta_.chunk_num + 1) * sizeof(uint32_t)),
(size_t)read_size);
return IndexError_ReadData;
}
memcpy(&(chunk_offsets[0]), data, read_size);
// load pq data
std::vector<uint8_t> pq_data;
pq_data_segment_ = storage_->get(DiskAnnEntity::kDiskAnnPqDataSegmentId);
if (!pq_data_segment_) {
LOG_ERROR("Miss or invalid segment %s",
DiskAnnEntity::kDiskAnnPqDataSegmentId.c_str());
return IndexError_InvalidFormat;
}
pq_data.resize(meta_header_.doc_cnt * pq_meta_.chunk_num);
void *pq_data_ptr = &pq_data[0];
read_size = pq_data_segment_->fetch(
0, pq_data_ptr, meta_header_.doc_cnt * pq_meta_.chunk_num);
if (read_size != meta_header_.doc_cnt * pq_meta_.chunk_num) {
LOG_ERROR("Read segment %s failed, expect: %zu, actual: %zu",
DiskAnnEntity::kDiskAnnPqMetaSegmentId.c_str(),
(size_t)(meta_header_.doc_cnt * pq_meta_.chunk_num),
(size_t)read_size);
return IndexError_ReadData;
}
pq_table_ = std::make_shared<PQTable>(meta_, pq_meta_.chunk_num);
pq_table_->init(full_pivot_data, centroid, chunk_offsets, pq_data);
return 0;
}
int DiskAnnSearcherEntity::load_header_segment() {
const void *data = nullptr;
meta_segment_ = storage_->get(kDiskAnnMetaSegmentId);
if (!meta_segment_ ||
meta_segment_->data_size() < sizeof(DiskAnnMetaHeader)) {
LOG_ERROR("Miss or invalid segment %s", kDiskAnnMetaSegmentId.c_str());
return IndexError_InvalidFormat;
}
if (meta_segment_->read(0, reinterpret_cast<const void **>(&data),
sizeof(DiskAnnMetaHeader)) !=
sizeof(DiskAnnMetaHeader)) {
LOG_ERROR("Read segment %s failed", kDiskAnnMetaSegmentId.c_str());
return IndexError_ReadData;
}
memcpy(reinterpret_cast<uint8_t *>(&meta_header_), data,
sizeof(DiskAnnMetaHeader));
return 0;
}
int DiskAnnSearcherEntity::load_vector_segment() {
vector_segment_ = storage_->get(kDiskAnnVectorSegmentId);
if (!vector_segment_) {
LOG_ERROR("Miss or invalid segment %s",
DiskAnnEntity::kDiskAnnVectorSegmentId.c_str());
return IndexError_InvalidFormat;
}
return 0;
}
int DiskAnnSearcherEntity::load_key_segment() {
// load key
key_segment_ = storage_->get(kDiskAnnKeySegmentId);
if (!key_segment_) {
LOG_ERROR("Miss or invalid segment %s",
DiskAnnEntity::kDiskAnnKeySegmentId.c_str());
return IndexError_InvalidFormat;
}
size_t key_data_len = doc_cnt() * sizeof(diskann_key_t);
const void *data = nullptr;
if (key_segment_->read(0, reinterpret_cast<const void **>(&data),
key_data_len) != key_data_len) {
LOG_ERROR("Read segment %s failed", kDiskAnnKeySegmentId.c_str());
return IndexError_ReadData;
}
key_buffer_.resize(key_data_len);
memcpy(&(key_buffer_[0]), data, key_data_len);
return 0;
}
int DiskAnnSearcherEntity::load_entrypoint_segment() {
entrypoint_segment_ = storage_->get(kDiskAnnEntryPointSegmentId);
if (!entrypoint_segment_) {
LOG_ERROR("Miss or invalid segment %s",
DiskAnnEntity::kDiskAnnEntryPointSegmentId.c_str());
return IndexError_InvalidFormat;
}
const void *data = nullptr;
if (entrypoint_segment_->read(0, reinterpret_cast<const void **>(&data),
sizeof(uint32_t)) != sizeof(uint32_t)) {
LOG_ERROR("Read segment %s failed", kDiskAnnEntryPointSegmentId.c_str());
return IndexError_ReadData;
}
uint32_t entrypoint_cnt = 0;
memcpy(&entrypoint_cnt, data, sizeof(uint32_t));
if (entrypoint_cnt != 0) {
size_t entrypoint_data_len = entrypoint_cnt * sizeof(diskann_id_t);
if (entrypoint_segment_->read(sizeof(uint32_t),
reinterpret_cast<const void **>(&data),
entrypoint_data_len) != entrypoint_data_len) {
LOG_ERROR("Read segment %s failed", kDiskAnnEntryPointSegmentId.c_str());
return IndexError_ReadData;
}
entrypoints_.resize(entrypoint_cnt);
memcpy(&(entrypoints_[0]), data, entrypoint_data_len);
}
return 0;
}
int DiskAnnSearcherEntity::load_key_mapping_segment() {
key_mapping_segment_ = storage_->get(kDiskAnnKeyMappingSegmentId);
if (!key_mapping_segment_) {
LOG_ERROR("Miss or invalid segment %s",
DiskAnnEntity::kDiskAnnKeyMappingSegmentId.c_str());
return IndexError_InvalidFormat;
}
size_t key_mapping_data_len = doc_cnt() * sizeof(diskann_id_t);
const void *data = nullptr;
if (key_mapping_segment_->read(0, reinterpret_cast<const void **>(&data),
key_mapping_data_len) !=
key_mapping_data_len) {
LOG_ERROR("Read segment %s failed", kDiskAnnKeyMappingSegmentId.c_str());
return IndexError_ReadData;
}
key_mapping_buffer_.resize(key_mapping_data_len);
memcpy(&(key_mapping_buffer_[0]), data, key_mapping_data_len);
return 0;
}
//! Get vector local id by key
diskann_id_t DiskAnnSearcherEntity::get_id(diskann_key_t key) const {
const diskann_id_t *key_mapping_data_ptr =
reinterpret_cast<const diskann_id_t *>(key_mapping_buffer_.data());
const diskann_key_t *key_data_ptr =
reinterpret_cast<const diskann_key_t *>(key_buffer_.data());
//! Do binary search
diskann_id_t start = 0UL;
diskann_id_t end = doc_cnt();
diskann_id_t idx = 0u;
while (start < end) {
idx = start + (end - start) / 2;
diskann_id_t local_id = key_mapping_data_ptr[idx];
const diskann_key_t local_key = key_data_ptr[local_id];
if (local_key < key) {
start = idx + 1;
} else if (local_key > key) {
end = idx;
} else {
return local_id;
}
}
return kInvalidId;
}
diskann_key_t DiskAnnSearcherEntity::get_key(diskann_id_t id) const {
const diskann_key_t *key_data_ptr =
reinterpret_cast<const diskann_key_t *>(key_buffer_.data());
return key_data_ptr[id];
}
const void *DiskAnnSearcherEntity::get_vector(diskann_id_t id) const {
if (!vector_segment_) {
LOG_ERROR("Vector segment is null");
return nullptr;
}
uint64_t sector_offset =
DiskAnnUtil::get_node_sector(node_per_sector(), max_node_size(),
DiskAnnUtil::kSectorSize, id) *
DiskAnnUtil::kSectorSize;
uint64_t within_sector_offset =
(node_per_sector() == 0 ? 0 : (id % node_per_sector()) * max_node_size());
uint64_t total_offset = sector_offset + within_sector_offset;
size_t read_size = meta_.element_size();
const void *vec;
if (ailego_unlikely(vector_segment_->read(total_offset, &vec, read_size) !=
read_size)) {
LOG_ERROR("Read vector from segment failed, id: %u, offset: %lu", id,
total_offset);
return nullptr;
}
return vec;
}
std::pair<uint32_t, const diskann_id_t *> DiskAnnSearcherEntity::get_neighbors(
diskann_id_t id) const {
if (!vector_segment_) {
return std::make_pair(0, nullptr);
}
uint64_t read_sector_offset =
DiskAnnUtil::get_node_sector(node_per_sector(), max_node_size(),
DiskAnnUtil::kSectorSize, id) *
DiskAnnUtil::kSectorSize;
uint64_t node_vec_offset =
read_sector_offset +
(node_per_sector() == 0 ? 0 : (id % node_per_sector()) * max_node_size());
const void *data;
if (ailego_unlikely(
vector_segment_->read(node_vec_offset, &data, max_node_size()) !=
max_node_size())) {
LOG_ERROR("Read neighbors from segment failed");
return {0, nullptr};
}
const uint8_t *data_ptr = reinterpret_cast<const uint8_t *>(data);
const diskann_id_t *node_neighbor =
reinterpret_cast<const diskann_id_t *>(data_ptr + meta_.element_size());
auto neighbor_num = *node_neighbor;
return std::make_pair(neighbor_num, node_neighbor + 1);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,125 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/ailego/parallel/thread_pool.h>
#include <zvec/core/framework/index_holder.h>
#include "diskann_entity.h"
#include "diskann_file_reader.h"
#include "diskann_pq_table.h"
namespace zvec {
namespace core {
class DiskAnnSearcherEntity : public DiskAnnEntity {
public:
using Pointer = std::shared_ptr<DiskAnnSearcherEntity>;
using SegmentPointer = IndexStorage::Segment::Pointer;
public:
DiskAnnSearcherEntity() = default;
virtual ~DiskAnnSearcherEntity() = default;
public:
virtual const DiskAnnEntity::Pointer clone() const override;
int load(const IndexMeta &meta, IndexStorage::Pointer storage);
int load_pq_segment();
int load_header_segment();
int load_vector_segment();
int load_key_segment();
int load_key_mapping_segment();
int load_entrypoint_segment();
PQTable::Pointer get_pq_table() {
return pq_table_;
}
IndexStorage::Pointer get_storage() {
return storage_;
}
SegmentPointer get_vector_segment() {
return vector_segment_;
}
std::vector<diskann_id_t> &entrypoints() {
return entrypoints_;
}
std::pair<uint32_t, const diskann_id_t *> get_neighbors(
diskann_id_t id) const override;
diskann_id_t get_id(diskann_key_t key) const override;
diskann_key_t get_key(diskann_id_t id) const override;
const void *get_vector(diskann_id_t id) const override;
private:
DiskAnnSearcherEntity(
const DiskAnnMetaHeader &meta_header, const DiskAnnPqMeta &pq_meta,
const SegmentPointer &meta_segment, const SegmentPointer &pq_meta_segment,
const SegmentPointer &pq_data_segment,
const SegmentPointer &vector_segment, const SegmentPointer &key_segment,
const SegmentPointer &key_mapping_segment,
const SegmentPointer &entrypoint_segment, uint32_t num_threads,
uint32_t list_size, uint32_t cache_nodes_num, bool warm_up,
uint32_t beam_size, const IndexMeta meta, PQTable::Pointer pq_table,
const std::string &key_buffer, const std::string &key_mapping_buffer,
const std::vector<diskann_id_t> &entrypoints)
: DiskAnnEntity(meta_header, pq_meta),
meta_segment_(meta_segment),
pq_meta_segment_(pq_meta_segment),
pq_data_segment_(pq_data_segment),
vector_segment_(vector_segment),
key_segment_(key_segment),
key_mapping_segment_(key_mapping_segment),
entrypoint_segment_{entrypoint_segment},
num_threads_{num_threads},
list_size_{list_size},
cache_nodes_num_{cache_nodes_num},
warm_up_{warm_up},
beam_size_{beam_size},
meta_{meta},
pq_table_{pq_table},
key_buffer_{key_buffer},
key_mapping_buffer_{key_mapping_buffer},
entrypoints_{entrypoints} {}
IndexStorage::Pointer storage_{};
SegmentPointer meta_segment_{nullptr};
SegmentPointer pq_meta_segment_{nullptr};
SegmentPointer pq_data_segment_{nullptr};
SegmentPointer vector_segment_{nullptr};
SegmentPointer key_segment_{nullptr};
SegmentPointer key_mapping_segment_{nullptr};
SegmentPointer entrypoint_segment_{nullptr};
uint32_t num_threads_{1};
uint32_t list_size_{200};
uint32_t cache_nodes_num_{0};
bool warm_up_{false};
uint32_t beam_size_{2};
IndexMeta meta_;
PQTable::Pointer pq_table_;
std::string key_buffer_;
std::string key_mapping_buffer_;
std::vector<diskann_id_t> entrypoints_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,345 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "diskann_streamer.h"
#include "diskann_context.h"
#include "diskann_index_provider.h"
#include "diskann_indexer.h"
#include "diskann_params.h"
namespace zvec {
namespace core {
DiskAnnStreamer::DiskAnnStreamer() {}
DiskAnnStreamer::~DiskAnnStreamer() {}
int DiskAnnStreamer::init(const IndexMeta &meta,
const ailego::Params &search_params) {
meta_ = meta;
search_params.get(PARAM_DISKANN_SEARCHER_LIST_SIZE, &list_size_);
search_params.get(PARAM_DISKANN_SEARCHER_CACHE_NODE_NUM, &cache_nodes_num_);
return 0;
}
void DiskAnnStreamer::print_debug_info() {}
int DiskAnnStreamer::cleanup() {
LOG_INFO("Begin DiskAnnStreamer:cleanup");
LOG_INFO("End DiskAnnStreamer:cleanup");
return 0;
}
int DiskAnnStreamer::open(IndexStorage::Pointer storage) {
LOG_INFO("DiskAnnStreamer::load Begin");
auto start_time = ailego::Monotime::MilliSeconds();
int ret = IndexHelper::DeserializeFromStorage(storage.get(), &meta_);
if (ret != 0) {
LOG_ERROR("Failed to deserialize meta from storage");
return ret;
}
ret = entity_.load(meta_, storage);
if (ret != 0) {
LOG_INFO("Searcher Entity Load Failed");
return ret;
}
diskann_indexer_ = std::make_shared<DiskAnnIndexer>(meta_);
int res = diskann_indexer_->init(entity_);
if (res != 0) {
return res;
}
if (cache_nodes_num_ != 0) {
std::vector<diskann_id_t> node_list;
LOG_INFO("Caching %u nodes around medoid(s)", cache_nodes_num_);
diskann_indexer_->cache_bfs_levels(cache_nodes_num_, node_list);
diskann_indexer_->load_cache_list(node_list);
node_list.clear();
node_list.shrink_to_fit();
}
measure_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!measure_) {
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
return IndexError_NoExist;
}
ret = measure_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
return ret;
}
if (measure_->query_metric()) {
measure_ = measure_->query_metric();
}
stats_.set_loaded_costtime(ailego::Monotime::MilliSeconds() - start_time);
state_ = STATE_LOADED;
magic_ = IndexContext::GenerateMagic();
LOG_INFO("DiskAnnStreamer::load Done");
return 0;
}
int DiskAnnStreamer::unload() {
LOG_INFO("DiskAnnStreamer unload index");
state_ = STATE_INITED;
return 0;
}
int DiskAnnStreamer::update_context(DiskAnnContext *ctx) const {
const DiskAnnEntity::Pointer entity = entity_.clone();
if (!entity) {
LOG_ERROR("Failed to clone search context entity");
return IndexError_Runtime;
}
return ctx->update_context(DiskAnnContext::kSearcherContext, meta_, measure_,
entity, magic_);
}
int DiskAnnStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const {
// do search
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to DiskAnnContext failed");
return IndexError_Cast;
}
// Context is pooled per index type. When switching between DiskAnn indexes
// with different element sizes (e.g., fp16 vs fp32), the cached context has
// undersized buffers. Recreate it to ensure correct buffer allocations.
if (ctx->magic() != magic_) {
uint32_t saved_topk = ctx->topk();
context = create_context();
if (!context) {
LOG_ERROR("Failed to recreate context for current streamer");
return IndexError_Runtime;
}
ctx = dynamic_cast<DiskAnnContext *>(context.get());
ctx->set_topk(saved_topk);
}
ctx->clear();
ctx->resize_results(count);
for (uint32_t i = 0; i < count; i++) {
ctx->reset_query(query);
diskann_indexer_->knn_search(ctx);
ctx->topk_to_result(i);
query = static_cast<const char *>(query) + qmeta.element_size();
}
return 0;
}
int DiskAnnStreamer::search_bf_impl(const void *query,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to DiskAnnContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer, recreate it
//! to ensure buffers are correctly sized for this index's parameters.
uint32_t saved_topk = ctx->topk();
context = create_context();
if (!context) {
LOG_ERROR("Failed to recreate context for current streamer");
return IndexError_Runtime;
}
ctx = dynamic_cast<DiskAnnContext *>(context.get());
ctx->set_topk(saved_topk);
}
ctx->clear();
ctx->resize_results(count);
for (size_t i = 0; i < count; ++i) {
ctx->reset_query(query);
diskann_indexer_->linear_search(ctx);
ctx->topk_to_result(i);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int DiskAnnStreamer::search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
DiskAnnContext *ctx = dynamic_cast<DiskAnnContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to DiskAnnContext failed");
return IndexError_Cast;
}
if (ailego_unlikely(p_keys.size() != count)) {
LOG_ERROR("The size of p_keys is not equal to count");
return IndexError_InvalidArgument;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer, recreate it
//! to ensure buffers are correctly sized for this index's parameters.
uint32_t saved_topk = ctx->topk();
context = create_context();
if (!context) {
LOG_ERROR("Failed to recreate context for current streamer");
return IndexError_Runtime;
}
ctx = dynamic_cast<DiskAnnContext *>(context.get());
ctx->set_topk(saved_topk);
}
ctx->clear();
ctx->resize_results(count);
for (size_t i = 0; i < count; ++i) {
ctx->reset_query(query);
diskann_indexer_->keys_search(p_keys[i], ctx);
ctx->topk_to_result(i);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int DiskAnnStreamer::get_vector(uint64_t key, Context::Pointer &context,
std::string &vector) const {
return diskann_indexer_->get_vector(key, context, vector);
}
const void *DiskAnnStreamer::get_vector_by_id(uint32_t id) const {
// DiskAnn vectors are stored on disk in sector format;
// a const void* access requires sector I/O via create_context
// Return nullptr to indicate this path is not supported.
return nullptr;
}
int DiskAnnStreamer::get_vector_by_id(const uint32_t id,
IndexStorage::MemoryBlock &block) const {
// Lazily create a reusable context for fetch operations
if (!fetch_ctx_) {
fetch_ctx_ = create_context();
if (!fetch_ctx_) {
LOG_ERROR("Failed to create context for get_vector_by_id");
return IndexError_Runtime;
}
}
int ret = diskann_indexer_->get_vector(id, fetch_ctx_, fetch_vector_buffer_);
if (ret != 0) {
LOG_ERROR("Failed to get vector by id: %u", id);
return IndexError_Runtime;
}
block.reset((void *)fetch_vector_buffer_.data());
return 0;
}
IndexSearcher::Provider::Pointer DiskAnnStreamer::create_provider(void) const {
if (state_ != STATE_LOADED) {
LOG_ERROR("Load the index first before creating a provider");
return nullptr;
}
const DiskAnnEntity::Pointer entity = entity_.clone();
if (!entity) {
LOG_ERROR("Failed to clone DiskAnn entity for provider");
return nullptr;
}
return IndexProvider::Pointer(new (std::nothrow) DiskAnnIndexProvider(
meta_, entity, "DiskAnnStreamer"));
}
IndexSearcher::Context::Pointer DiskAnnStreamer::create_context() const {
const DiskAnnEntity::Pointer search_ctx_entity = entity_.clone();
if (!search_ctx_entity) {
LOG_ERROR("Failed to create search context entity");
return Context::Pointer();
}
DiskAnnContext *ctx =
new (std::nothrow) DiskAnnContext(meta_, measure_, search_ctx_entity);
if (ctx == nullptr) {
LOG_ERROR("Failed to allocate DiskAnn Context");
return Context::Pointer();
}
if (ailego_unlikely(ctx->init(
DiskAnnContext::kSearcherContext, search_ctx_entity->max_degree(),
search_ctx_entity->pq_chunk_num(), meta_.element_size())) != 0) {
LOG_ERROR("Init DiskAnn Context failed");
delete ctx;
return Context::Pointer();
}
ctx->set_list_size(list_size_);
return Context::Pointer(ctx);
}
INDEX_FACTORY_REGISTER_STREAMER(DiskAnnStreamer);
} // namespace core
} // namespace zvec
@@ -0,0 +1,180 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/core/framework/index_framework.h>
#include "diskann_context.h"
#include "diskann_indexer.h"
class LinuxAlignedFileReader;
namespace zvec {
namespace core {
class DiskAnnStreamer : public IndexStreamer {
public:
using ContextPointer = IndexStreamer::Context::Pointer;
public:
DiskAnnStreamer(void);
~DiskAnnStreamer(void);
DiskAnnStreamer(const DiskAnnStreamer &) = delete;
DiskAnnStreamer &operator=(const DiskAnnStreamer &) = delete;
protected:
//! Initialize Searcher
int init(const IndexMeta &meta, const ailego::Params &params) override;
//! Cleanup Searcher
int cleanup(void) override;
//! Load Index from storage
int open(IndexStorage::Pointer storage) override;
//! Unload index from storage
int unload(void) override;
//! KNN Search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_impl(query, qmeta, 1, context);
}
//! KNN Search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, ContextPointer &context) const override;
//! Linear Search
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_impl(query, qmeta, 1, context);
}
//! Linear Search
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, ContextPointer &context) const override;
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query, const uint32_t sparse_count,
const uint32_t *sparse_indices,
const void *sparse_query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, &sparse_count, sparse_indices,
sparse_query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
int search_bf_by_p_keys_impl(
const void * /*query*/, const uint32_t * /*sparse_count*/,
const uint32_t * /*sparse_indices*/, const void * /*sparse_query*/,
const std::vector<std::vector<uint64_t>> & /*p_keys*/,
const IndexQueryMeta & /*qmeta*/, uint32_t /*count*/,
ContextPointer & /*context*/) const override {
return IndexError_NotImplemented;
}
//! Get vector by key
int get_vector(uint64_t key, Context::Pointer &context,
std::string &vector) const override;
//! Fetch vector by id
const void *get_vector_by_id(uint32_t id) const override;
//! Fetch vector by id into memory block
int get_vector_by_id(const uint32_t id,
IndexStorage::MemoryBlock &block) const override;
//! Create a searcher context
ContextPointer create_context() const override;
//! Create a vector iterator backed by the on-disk vector segment.
//! Used by the merge code path (``MixedStreamerReducer``) to walk every
//! vector held by this streamer.
IndexSearcher::Provider::Pointer create_provider(void) const override;
//! Retrieve statistics
const Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
const IndexMeta &meta(void) const override {
return meta_;
}
virtual int flush(uint64_t /*check_point*/) override {
return 0;
}
virtual int close(void) override {
return this->unload();
}
void print_debug_info() override;
private:
template <typename T, typename LabelT = uint32_t>
int search_disk_index(const std::string &query_file,
const uint32_t num_nodes_to_cache,
const uint32_t recall_at, const uint32_t beamwidth);
//! To share ctx across streamer/searcher, we need to update the context for
//! current streamer/searcher
int update_context(DiskAnnContext *ctx) const;
private:
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
IndexMetric::Pointer measure_{};
IndexMeta meta_{};
ailego::Params params_{};
uint32_t list_size_{200};
uint32_t cache_nodes_num_{0};
bool warm_up_{false};
uint32_t beam_size_{2};
DiskAnnIndexer::Pointer diskann_indexer_{nullptr};
DiskAnnSearcherEntity entity_{};
// Mutable members for get_vector_by_id (caches context and buffer)
mutable ContextPointer fetch_ctx_{};
mutable std::string fetch_vector_buffer_;
uint32_t magic_{0U};
Stats stats_;
State state_{STATE_INIT};
};
} // namespace core
} // namespace zvec
+221
View File
@@ -0,0 +1,221 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <iostream>
#include <zvec/core/framework/index_framework.h>
#include "diskann_entity.h"
namespace zvec {
namespace core {
class DiskAnnUtil {
public:
static constexpr uint64_t kSectorSize = 4096;
static constexpr uint64_t kMaxSectorReadNum = 128;
public:
static inline size_t div_round_up(size_t x, size_t y) {
return (x / y + (x % y != 0));
}
static inline size_t round_up(size_t x, size_t y) {
return div_round_up(x, y) * y;
}
static inline void alloc_aligned(void **ptr, size_t size, size_t align) {
*ptr = ::aligned_alloc(align, size);
}
static inline void free_aligned(void *ptr) {
if (ptr == nullptr) {
return;
}
free(ptr);
}
template <typename T>
static inline void convert_vector_to_residual(T *data, uint32_t blocksize_,
size_t dim, void *centroid) {
const T *centroid_ptr = reinterpret_cast<const T *>(centroid);
for (size_t i = 0; i < blocksize_; i++) {
for (uint64_t d = 0; d < dim; d++) {
float data_float = data[i * dim + d];
data_float -= centroid_ptr[d];
data[i * dim + d] = data_float;
}
}
}
static inline void convert_types_uint32_to_uint8(const uint32_t *src,
uint8_t *dest, size_t npts,
size_t dim) {
for (size_t i = 0; i < npts; i++) {
for (size_t j = 0; j < dim; j++) {
dest[i * dim + j] = src[i * dim + j];
}
}
}
static inline uint64_t get_node_sector(uint32_t node_per_sector,
uint32_t max_nodesize_,
uint32_t sectorsize_,
diskann_id_t node_id) {
return (node_per_sector > 0
? node_id / node_per_sector
: node_id * div_round_up(max_nodesize_, sectorsize_));
}
static inline uint32_t *offset_to_node_neighbor(uint8_t *node_buf,
uint32_t elementsize_) {
return (uint32_t *)(node_buf + elementsize_);
}
static inline uint8_t *offset_to_node(uint32_t node_per_sector,
uint32_t max_nodesize_,
uint8_t *sector_buf,
diskann_id_t node_id) {
return sector_buf + (node_per_sector == 0
? 0
: (node_id % node_per_sector) * max_nodesize_);
}
static inline const uint8_t *offset_to_node_const(uint32_t node_per_sector,
uint32_t max_nodesize_,
const uint8_t *sector_buf,
diskann_id_t node_id) {
return sector_buf + (node_per_sector == 0
? 0
: (node_id % node_per_sector) * max_nodesize_);
}
};
//! Neighbor
struct Neighbor {
public:
Neighbor() = default;
Neighbor(diskann_id_t id, float distance)
: id{id}, distance{distance}, expanded(false) {}
inline bool operator<(const Neighbor &other) const {
return distance < other.distance ||
(distance == other.distance && id < other.id);
}
inline bool operator==(const Neighbor &other) const {
return (id == other.id);
}
public:
diskann_id_t id;
float distance;
bool expanded;
};
//! NeighborPriorityQueue
class NeighborPriorityQueue {
public:
NeighborPriorityQueue() : size_(0), capacity_(0), cur_(0) {}
explicit NeighborPriorityQueue(size_t capacity)
: size_(0), capacity_(capacity), cur_(0), data_(capacity + 1) {}
void insert(const Neighbor &nbr) {
if (size_ == capacity_ && data_[size_ - 1] < nbr) {
return;
}
size_t low = 0, high = size_;
while (low < high) {
size_t mid = (low + high) >> 1;
if (nbr < data_[mid]) {
high = mid;
} else if (data_[mid].id == nbr.id) {
return;
} else {
low = mid + 1;
}
}
if (low < capacity_) {
std::memmove(&data_[low + 1], &data_[low],
(size_ - low) * sizeof(Neighbor));
}
data_[low] = {nbr.id, nbr.distance};
if (size_ < capacity_) {
size_++;
}
if (low < cur_) {
cur_ = low;
}
}
Neighbor closest_unexpanded() {
data_[cur_].expanded = true;
size_t pre = cur_;
while (cur_ < size_ && data_[cur_].expanded) {
cur_++;
}
return data_[pre];
}
bool has_unexpanded_node() const {
return cur_ < size_;
}
size_t size() const {
return size_;
}
size_t capacity() const {
return capacity_;
}
void reserve(size_t capacity) {
if (capacity + 1 > data_.size()) {
data_.resize(capacity + 1);
}
capacity_ = capacity;
}
Neighbor &operator[](size_t i) {
return data_[i];
}
Neighbor operator[](size_t i) const {
return data_[i];
}
void sort() {
std::sort(data_.begin(), data_.begin() + size_);
}
void clear() {
size_ = 0;
cur_ = 0;
}
private:
size_t size_;
size_t capacity_;
size_t cur_;
std::vector<Neighbor> data_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,320 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <iostream>
#include <ailego/io/mmap_file.h>
#include <framework/index_framework.h>
#include <zvec/core/framework/index_holder.h>
namespace zvec {
namespace core {
enum VecsBitMapIndex {
BITMAP_INDEX_KEY = 0,
BITMAP_INDEX_DENSE = 1,
BITMAP_INDEX_SPARSE = 2,
BITMAP_INDEX_PARTITION = 3,
BITMAP_INDEX_TAGLIST = 4
};
#pragma pack(4)
struct VecsHeaderDetect {
uint64_t num_vecs;
uint16_t reserved; // in v0, it's meta size
uint16_t version; // 0 for v0 format, 1 for new format
uint8_t meta_buf[0];
};
#pragma pack()
#pragma pack(4)
struct VecsHeader {
uint64_t num_vecs;
uint32_t meta_size;
uint8_t meta_buf[0];
};
#pragma pack()
#pragma pack(4)
struct VecsHeaderV1 {
uint64_t num_vecs;
uint16_t meta_size_v1;
uint16_t version;
uint32_t meta_size;
uint64_t bitmap; // set for data section
uint64_t key_offset; // offset for key
uint64_t key_size; // size for key
uint64_t dense_offset; // offset for dense
uint64_t dense_size; // size for dense
uint64_t sparse_offset; // offset for sparse
uint64_t sparse_size; // size for sparse
uint64_t partition_offset; // offset for partition
uint64_t partition_size; // size for partition
uint64_t taglist_offset; // offset for taglist
uint64_t taglist_size; // size for taglist
uint8_t meta_buf[0];
};
#pragma pack()
class DiskAnnVecsReader {
public:
DiskAnnVecsReader()
: mmap_file_(),
index_meta_(),
num_vecs_(0),
vector_base_(nullptr),
key_base_(nullptr),
sparse_base_meta_{nullptr},
sparse_base_data_{nullptr},
partition_base_{nullptr},
taglist_base_meta_{nullptr},
taglist_base_data_{nullptr},
taglist_size_{0} {}
void set_measure(const std::string &name, const IndexParams &params) {
index_meta_.set_measure(name, 0, params);
}
bool load(const std::string &fname) {
return load(fname.c_str());
}
bool load(const char *fname) {
if (!fname) {
std::cerr << "Load fname is nullptr" << std::endl;
return false;
}
if (!mmap_file_.open(fname, true)) {
std::cerr << "Open file error: " << fname << std::endl;
return false;
}
if (mmap_file_.size() < sizeof(VecsHeaderDetect)) {
std::cerr << "File size is too small: " << mmap_file_.size() << std::endl;
return false;
}
const VecsHeaderDetect *header =
reinterpret_cast<const VecsHeaderDetect *>(mmap_file_.region());
if (header->version == 0) {
return load_v0();
} else if (header->version == 1) {
return load_v1();
}
std::cerr << "Can not recognize version: " << header->version << std::endl;
return false;
}
bool load_v0() {
const VecsHeader *header =
reinterpret_cast<const VecsHeader *>(mmap_file_.region());
// check
num_vecs_ = header->num_vecs;
// deserialize
bool bret = index_meta_.deserialize(&header->meta_buf, header->meta_size);
if (!bret) {
std::cerr << "deserialize index meta error." << std::endl;
return false;
}
if (!index_meta_.hybrid_vector()) {
if ((mmap_file_.size() - sizeof(*header) - header->meta_size) %
num_vecs_ !=
0) {
std::cerr << "input file foramt check error." << std::endl;
return false;
}
}
if (!index_meta_.hybrid_vector()) {
vector_base_ =
reinterpret_cast<const char *>(header + 1) + header->meta_size;
key_base_ = reinterpret_cast<const uint64_t *>(
vector_base_ + num_vecs_ * index_meta_.element_size());
} else {
vector_base_ =
reinterpret_cast<const char *>(header + 1) + header->meta_size;
key_base_ = reinterpret_cast<const uint64_t *>(
vector_base_ + num_vecs_ * index_meta_.element_size());
sparse_base_meta_ = reinterpret_cast<const char *>(key_base_ + num_vecs_);
sparse_base_data_ = reinterpret_cast<const char *>(
sparse_base_meta_ + num_vecs_ * sizeof(uint64_t));
}
return true;
}
bool load_v1() {
const VecsHeaderV1 *header =
reinterpret_cast<const VecsHeaderV1 *>(mmap_file_.region());
// check
num_vecs_ = header->num_vecs;
// deserialize
bool bret = index_meta_.deserialize(&header->meta_buf, header->meta_size);
if (!bret) {
std::cerr << "deserialize index meta error." << std::endl;
return false;
}
const char *data_base_ptr =
reinterpret_cast<const char *>(header + 1) + header->meta_size;
vector_base_ = reinterpret_cast<const char *>(data_base_ptr);
key_base_ = reinterpret_cast<const uint64_t *>(
vector_base_ + num_vecs_ * index_meta_.element_size());
if (header->sparse_offset != -1LLU) {
sparse_base_meta_ = data_base_ptr + header->sparse_offset;
sparse_base_data_ = sparse_base_meta_ + num_vecs_ * sizeof(uint64_t);
}
if (header->partition_offset != -1LLU) {
partition_base_ = reinterpret_cast<const uint32_t *>(
data_base_ptr + header->partition_offset);
}
if (header->taglist_offset != -1LLU) {
taglist_base_meta_ = data_base_ptr + header->taglist_offset;
taglist_base_data_ = taglist_base_meta_ + num_vecs_;
taglist_size_ = header->taglist_size;
}
return true;
}
size_t num_vecs() const {
return num_vecs_;
}
const void *vector_base() const {
return vector_base_;
}
const uint64_t *key_base() const {
return key_base_;
}
const IndexMeta &index_meta() const {
return index_meta_;
}
uint64_t get_key(size_t index) const {
return key_base_[index];
}
const void *get_vector(size_t index) const {
return vector_base_ + index * index_meta_.element_size();
}
uint32_t get_sparse_count(size_t index) const {
if (index_meta_.hybrid_vector()) {
auto sparse_data_meta = sparse_base_meta_ + index * sizeof(uint64_t);
uint64_t sparse_offset = *((uint64_t *)sparse_data_meta);
uint32_t sparse_count =
*((uint32_t *)(sparse_base_data_ + sparse_offset));
return sparse_count;
}
return 0;
}
const uint32_t *get_sparse_indices(size_t index) const {
if (index_meta_.hybrid_vector()) {
auto sparse_data_meta = sparse_base_meta_ + index * sizeof(uint64_t);
uint64_t sparse_offset = *((uint64_t *)sparse_data_meta);
uint32_t *sparse_indices =
(uint32_t *)(sparse_base_data_ + sparse_offset + sizeof(uint32_t));
return sparse_indices;
}
return nullptr;
}
const void *get_sparse_data(size_t index) const {
if (index_meta_.hybrid_vector()) {
auto sparse_data_meta = sparse_base_meta_ + index * sizeof(uint64_t);
uint64_t sparse_offset = *((uint64_t *)sparse_data_meta);
uint32_t sparse_count =
*((uint32_t *)(sparse_base_data_ + sparse_offset));
void *sparse_data =
(uint32_t *)(sparse_base_data_ + sparse_offset + sizeof(uint32_t) +
sparse_count * sizeof(uint32_t));
return sparse_data;
}
return nullptr;
}
size_t get_total_sparse_count(void) const {
size_t total_sparse_count = 0;
for (size_t i = 0; i < num_vecs_; ++i) {
total_sparse_count += get_sparse_count(i);
}
return total_sparse_count;
}
bool has_taglist(void) const {
return taglist_base_meta_ != nullptr;
}
uint64_t get_taglist_count(size_t index) const {
if (!taglist_base_data_ || !taglist_base_meta_) {
return 0;
}
uint64_t taglist_count = *reinterpret_cast<const uint64_t *>(
taglist_base_data_ + taglist_base_meta_[index]);
return taglist_count;
}
const uint64_t *get_taglist(size_t index) const {
if (!taglist_base_data_ || !taglist_base_meta_) {
return nullptr;
}
return reinterpret_cast<const uint64_t *>(taglist_base_data_ +
taglist_base_meta_[index]) +
1;
}
const void *get_taglist_data(size_t &size) const {
size = taglist_size_;
return taglist_base_meta_;
}
private:
ailego::MMapFile mmap_file_;
IndexMeta index_meta_;
size_t num_vecs_;
const char *vector_base_;
const uint64_t *key_base_;
const char *sparse_base_meta_;
const char *sparse_base_data_;
const uint32_t *partition_base_;
const char *taglist_base_meta_;
const char *taglist_base_data_;
uint64_t taglist_size_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,419 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <algorithm>
#include <chrono>
#include <cstdint>
#include <limits>
#include <random>
#include <tuple>
#include <ailego/container/bloom_filter.h>
#include <ailego/utility/bitset_helper.h>
#include <zvec/core/framework/index_error.h>
#include <zvec/core/framework/index_logger.h>
namespace zvec {
namespace core {
struct VisitFilterHeader {
VisitFilterHeader() : maxDocCnt(0), maxScanNum(0) {}
uint64_t maxDocCnt;
uint64_t maxScanNum;
};
constexpr int PROXIMA_DISKANN_VISITFILTER_CUSTOM_PARAMS_INDEX_NEGPROB = 0;
class VisitBloomFilter {
public:
static constexpr int mode = 1;
static constexpr int N = 5;
struct Context {
Context()
: mt(std::chrono::system_clock::now().time_since_epoch().count()) {};
VisitFilterHeader h;
std::mt19937 mt;
ailego::BloomFilter<N> *filter{nullptr};
int offset[N] = {0};
};
#define BLOOM_FILTER_HASH_BITS_OFFSETS(i) \
i + c->offset[0], i + c->offset[1], i + c->offset[2], i + c->offset[3], \
i + c->offset[4]
VisitBloomFilter() = delete;
inline static void set_visited(Context *c, id_t idx) {
c->filter->force_insert(BLOOM_FILTER_HASH_BITS_OFFSETS(idx));
return;
}
inline static void *get_visited(Context *, id_t) {
// TODO
return nullptr;
}
inline static bool visited(Context *c, id_t idx) {
return c->filter->has(BLOOM_FILTER_HASH_BITS_OFFSETS(idx));
}
inline static int set_max_scan_num(Context *c, uint64_t maxScanNum) {
if (maxScanNum == c->h.maxScanNum) {
return 0;
}
c->h.maxScanNum = maxScanNum;
if (c->filter->reset(maxScanNum, c->filter->probability()) != 0) {
LOG_ERROR("reset BloomFilter failed");
return IndexError_Runtime;
}
genRandomHashBits(c);
return 0;
}
inline static void clear(Context *c) {
c->filter->clear();
return;
}
inline static bool reset(Context *c, uint64_t maxDocCnt,
uint64_t max_scan_num) {
if (ailego_unlikely(maxDocCnt > c->h.maxDocCnt ||
max_scan_num > c->h.maxScanNum)) {
// Create a new one, if failed, we can reuse the old one
auto filter = new (std::nothrow) ailego::BloomFilter<VisitBloomFilter::N>(
max_scan_num, c->filter->probability());
if (ailego_unlikely(filter == nullptr)) {
LOG_ERROR("reset bloomfilter failed, maxScanNum %zu prob %f",
(size_t)max_scan_num, c->filter->probability());
c->filter->clear();
return false;
}
delete c->filter;
c->filter = filter;
c->h.maxScanNum = max_scan_num;
c->h.maxDocCnt = maxDocCnt;
genRandomHashBits(c);
}
return true;
}
inline static void genRandomHashBits(Context *c) {
std::uniform_int_distribution<int> dt(0, c->h.maxDocCnt);
for (size_t i = 0; i < sizeof(c->offset) / sizeof(c->offset[0]); ++i) {
int r = dt(c->mt);
size_t j = 0;
do { // gen distinct number
for (j = 0; j < i; ++j) {
if (c->offset[j] == r) {
r = dt(c->mt);
break;
}
}
} while (j < i);
c->offset[i] = r;
}
std::sort(c->offset, c->offset + N);
}
template <class... T>
static int init(Context *, void **ctx, uint64_t maxDocCnt,
uint64_t maxScanNum, std::tuple<T...> &&tpl) {
Context *c = new (std::nothrow) Context;
if (c == nullptr) {
LOG_ERROR("New memory in initVisitBitMap failed");
return IndexError_NoMemory;
}
c->h.maxDocCnt = maxDocCnt;
c->h.maxScanNum = maxScanNum;
float p =
std::get<PROXIMA_DISKANN_VISITFILTER_CUSTOM_PARAMS_INDEX_NEGPROB>(tpl);
c->filter = new (std::nothrow)
ailego::BloomFilter<VisitBloomFilter::N>(maxScanNum, p);
if (c->filter == nullptr) {
LOG_ERROR("New BloomFilter failed, reuse old one");
return IndexError_NoMemory;
}
genRandomHashBits(c);
*ctx = c;
return 0;
}
inline static void destroy(Context *c) {
delete c->filter;
delete c;
}
#undef BLOOM_FILTER_HASH_BITS_OFFSETS
}; // end of VisitBloomFilter
class VisitBitMap {
public:
static constexpr int mode = 2;
struct Context {
VisitFilterHeader h;
ailego::BitsetHelper bitset;
char *buf{nullptr};
};
VisitBitMap() = delete;
inline static void set_visited(Context *c, id_t idx) {
c->bitset.set(idx);
return;
}
inline static void *get_visited(Context *c, id_t idx) {
return &c->buf[idx >> 3];
}
inline static bool visited(Context *c, id_t idx) {
return c->bitset.test(idx);
}
inline static int set_max_scan_num(Context *c, uint64_t maxScanNum) {
c->h.maxScanNum = maxScanNum;
return 0;
}
inline static void clear(Context *c) {
c->bitset.clear();
return;
}
inline static bool reset(Context *c, uint64_t maxDocCnt,
uint64_t maxScanNum) {
if (ailego_unlikely(maxDocCnt > c->h.maxDocCnt ||
maxScanNum > c->h.maxScanNum)) {
uint64_t len = ((maxDocCnt + 31) >> 5) << 2; // round to uint32_t
auto buf = new (std::nothrow) char[len];
if (buf == nullptr) {
LOG_ERROR("New memory in initVisitBitMap failed");
c->bitset.clear();
return false;
}
c->h.maxDocCnt = maxDocCnt;
c->h.maxScanNum = maxScanNum;
delete[] c->buf;
c->buf = buf;
memset(c->buf, 0, len);
c->bitset.mount(c->buf, len);
}
return true;
}
template <class... T>
static int init(Context *, void **ctx, uint64_t maxDocCnt,
uint64_t maxScanNum, std::tuple<T...> &&tpl) {
(void)tpl; // unsed warning
Context *c = new (std::nothrow) Context;
if (c == nullptr) {
LOG_ERROR("New memory in initVisitBitMap failed");
return IndexError_NoMemory;
}
c->h.maxDocCnt = maxDocCnt;
c->h.maxScanNum = maxScanNum;
uint64_t len = ((maxDocCnt + 31) >> 5) << 2; // round to uint32_t
c->buf = new (std::nothrow) char[len];
if (c->buf == nullptr) {
LOG_ERROR("New memory in initVisitBitMap failed, reuse old one");
delete c;
return IndexError_NoMemory;
}
memset(c->buf, 0, len);
c->bitset.mount(c->buf, len);
*ctx = c;
return 0;
}
inline static void destroy(Context *c) {
delete[] c->buf;
delete c;
}
}; // end of VisitBitMap
class VisitByteMap {
public:
static constexpr int mode = 3;
struct Context {
VisitFilterHeader h;
uint8_t curNum{0};
uint8_t *arr{nullptr};
};
VisitByteMap() = delete;
inline static void set_visited(Context *c, id_t idx) {
c->arr[idx] = c->curNum;
return;
}
inline static void *get_visited(Context *c, id_t idx) {
return c->arr + idx;
}
inline static bool visited(Context *c, id_t idx) {
return c->arr[idx] == c->curNum;
}
inline static int set_max_scan_num(Context *c, uint64_t maxScanNum) {
c->h.maxScanNum = maxScanNum;
return 0;
}
inline static void clear(Context *c) {
c->curNum++;
if (c->curNum == 0) {
memset(c->arr, 0, c->h.maxDocCnt * sizeof(uint8_t));
c->curNum = 1;
}
return;
}
inline static bool reset(Context *c, uint64_t maxDocCnt,
uint64_t maxScanNum) {
if (ailego_unlikely(maxDocCnt > c->h.maxDocCnt ||
maxScanNum > c->h.maxScanNum)) {
auto arr = new (std::nothrow) uint8_t[maxDocCnt];
if (arr != nullptr) {
memset(arr, 0, maxDocCnt * sizeof(uint8_t));
c->curNum = 1;
c->h.maxDocCnt = maxDocCnt;
c->h.maxScanNum = maxScanNum;
delete[] c->arr;
c->arr = arr;
return true;
}
LOG_ERROR("New memory in initVisitByteMap failed, reuse old one");
}
return true;
}
template <class... T>
static int init(Context *, void **ctx, uint64_t maxDocCnt,
uint64_t maxScanNum, std::tuple<T...> &&tpl) {
(void)tpl; // unsed warning
Context *c = new (std::nothrow) Context;
if (c == nullptr) {
LOG_ERROR("New memory in initVisitByteMap failed");
return IndexError_NoMemory;
}
c->h.maxDocCnt = maxDocCnt;
c->h.maxScanNum = maxScanNum;
c->arr = new (std::nothrow) uint8_t[maxDocCnt];
if (c->arr == nullptr) {
LOG_ERROR("New memory in initVisitByteMap failed");
delete c;
return IndexError_NoMemory;
}
memset(c->arr, 0, maxDocCnt * sizeof(uint8_t));
c->curNum = 1;
*ctx = c;
return 0;
}
inline static void destroy(Context *c) {
delete[] c->arr;
delete c;
}
}; // end of VisitByteMap
#define PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(cls, impl, ctx, ...) \
case cls::mode: \
return cls::impl(static_cast<cls::Context *>(ctx), ##__VA_ARGS__);
#define PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(impl, ...) \
switch (mode_) { \
PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(VisitBloomFilter, impl, ctx_, \
##__VA_ARGS__) \
PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(VisitBitMap, impl, ctx_, \
##__VA_ARGS__) \
PROXIMA_DISKANN_VISITFILTER_SWITCH_CASE(VisitByteMap, impl, ctx_, \
##__VA_ARGS__) \
}
// visit list will be called with high frequency,
// so using switch instead of std::function or virtual class
// funtion point, lambda, virtual class all cannot be inlined
class VisitFilter {
public:
enum Mode {
Default = 0,
BloomFilter = VisitBloomFilter::mode,
BitMap = VisitBitMap::mode,
ByteMap = VisitByteMap::mode
};
VisitFilter() : mode_(0), ctx_(nullptr) {};
inline bool visited(id_t idx) {
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(visited, idx);
return true; // place holder
}
inline void set_visited(id_t idx) {
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(set_visited, idx);
}
inline void *get_visited(id_t idx) {
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(get_visited, idx);
return nullptr; // place holder
}
inline int set_max_scan_num(id_t idx) {
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(set_max_scan_num, idx);
return 0; // place holder
}
inline void clear() {
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(clear);
}
inline bool reset(uint64_t maxDocCnt, uint64_t maxScanNum) {
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(reset, maxDocCnt, maxScanNum);
return true;
}
inline void destroy() {
if (ctx_ != nullptr) {
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(destroy);
}
}
int init(int mode, uint64_t maxDocCnt, uint64_t maxScanNum,
float negativeProbility) {
mode_ = mode;
PROXIMA_DISKANN_VISITFILTER_CALL_IMPL(init, &ctx_, maxDocCnt, maxScanNum,
std::make_tuple(negativeProbility));
return 0; // place holder
}
int get_mode(void) const {
return mode_;
}
private:
VisitFilter(const VisitFilter &) = delete;
VisitFilter &operator=(const VisitFilter &) = delete;
int mode_{0U}; // custom data for each method
void *ctx_{nullptr};
};
} // namespace core
} // namespace zvec
+1 -1
View File
@@ -533,4 +533,4 @@ class HnswContext : public IndexContext {
};
} // namespace core
} // namespace zvec
} // namespace zvec