feat: add hnsw-rabitq support (#69)

* feat: add hnsw-rabitq

* undefine transform

* support config rotator type

* support sample_count

* fix ut

* refactor: update interface

* refactor: update interface

* update rabitq index params

* fix interface update

* fix searcher test

* fix streamer test

* streamer support bf and add more ut

* rm env check

* add collection ut

* add schema check

* add rabitq query param binding

* add integration test

* fix local_builder

* cleanup dist calculator

* add RaBitQ-Library submodule

* cleanup rabitq converter/reformer

* add files

* disable build on mac

* disable Feature_Optimize_HNSW_RABITQ

* disable python/tests/test_collection_hnsw_rabitq.py:13

* check avx2/avx512

* fix refine

* fix mac ci

* add dimension check

* fix mac ci

* fix ci

* add missing lib

* fix centroids selection for Cosine/InnerProduct

* rename hnsw-rabitq to hnsw_rabitq

* address comments

* fix compile

* fix rabitqlib name

* fix mac compile

* check avx2/avx512 support

* runtime check again compile-time

* check AUTO_DETECT_ARCH

* rm debug log

* fix

* rabitq use avx2 by default

* add missing file

* fix search_bf/group_by dist

* address comments

* fix typo

* address comments

* rabitq support disable id_map

* address comments
This commit is contained in:
egolearner
2026-03-19 17:21:42 +08:00
committed by GitHub
parent 432de52c5f
commit e5ba11b6fe
96 changed files with 14500 additions and 92 deletions
+22 -1
View File
@@ -6,4 +6,25 @@ cc_directory(flat)
cc_directory(flat_sparse)
cc_directory(ivf)
cc_directory(hnsw)
cc_directory(hnsw_sparse)
cc_directory(hnsw_sparse)
if(RABITQ_SUPPORTED)
message(STATUS "BUILD RABITQ")
cc_directory(hnsw_rabitq)
else()
message(STATUS "NOT BUILD RABITQ")
# Empty stub library for unsupported platforms
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/rabitq_stub.cc
"// Stub implementation for unsupported platforms\n"
"// RaBitQ only supports Linux x86_64\n"
"namespace zvec { namespace core { /* empty namespace for compatibility */ } }\n"
)
cc_library(
NAME core_knn_hnsw_rabitq
STATIC SHARED STRICT ALWAYS_LINK
SRCS ${CMAKE_CURRENT_BINARY_DIR}/rabitq_stub.cc
LIBS core_framework
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
VERSION "${PROXIMA_ZVEC_VERSION}"
)
endif()
@@ -1250,7 +1250,7 @@ int OptKmeansCluster::init(const IndexMeta &meta,
const ailego::Params &params) {
auto type_ = meta.data_type();
if (meta.metric_name() == "InnerProduct") {
if (meta.metric_name() == "InnerProduct" || meta.metric_name() == "Cosine") {
switch (type_) {
case IndexMeta::DataType::DT_FP16: {
algorithm_.reset(
-1
View File
@@ -127,7 +127,6 @@ int FlatBuilder<BATCH_SIZE>::dump(const IndexDumper::Pointer &dumper) {
return error_code;
}
holder_ = nullptr;
stats_.set_dumped_count(keys.size());
stats_.set_dumped_costtime(stamp.milli_seconds());
return 0;
@@ -0,0 +1,21 @@
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
if(AUTO_DETECT_ARCH)
foreach(FILE ${HNSW_RABITQ_FILES})
set_source_files_properties(
${FILE}
PROPERTIES
COMPILE_FLAGS "${RABITQ_ARCH_FLAG}"
)
endforeach()
endif()
cc_library(
NAME core_knn_hnsw_rabitq
STATIC SHARED STRICT ALWAYS_LINK
SRCS *.cc
LIBS core_framework rabitqlib sparsehash
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
VERSION "${PROXIMA_ZVEC_VERSION}"
)
@@ -0,0 +1,382 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_algorithm.h"
#include <chrono>
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
HnswRabitqAlgorithm::HnswRabitqAlgorithm(HnswRabitqEntity &entity)
: entity_(entity),
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
lock_pool_(kLockCnt) {}
int HnswRabitqAlgorithm::cleanup() {
return 0;
}
int HnswRabitqAlgorithm::add_node(node_id_t id, level_t level,
HnswRabitqContext *ctx) {
spin_lock_.lock();
// std::cout << "id: " << id << ", level: " << level << std::endl;
auto cur_max_level = entity_.cur_max_level();
auto entry_point = entity_.entry_point();
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
entity_.update_ep_and_level(id, level);
spin_lock_.unlock();
return 0;
}
spin_lock_.unlock();
if (ailego_unlikely(level > cur_max_level)) {
mutex_.lock();
// re-check max level
cur_max_level = entity_.cur_max_level();
entry_point = entity_.entry_point();
if (level <= cur_max_level) {
mutex_.unlock();
}
}
level_t cur_level = cur_max_level;
ResultRecord dist = ctx->dist_calculator()(entry_point);
for (; cur_level > level; --cur_level) {
select_entry_point(cur_level, &entry_point, &dist, ctx);
}
for (; cur_level >= 0; --cur_level) {
search_neighbors(cur_level, &entry_point, &dist, ctx->level_topk(cur_level),
ctx);
}
// add neighbors from down level to top level, to avoid upper level visible
// to knn_search but the under layer level not ready
for (cur_level = 0; cur_level <= level; ++cur_level) {
add_neighbors(id, cur_level, ctx->level_topk(cur_level), ctx);
ctx->level_topk(cur_level).clear();
}
if (ailego_unlikely(level > cur_max_level)) {
spin_lock_.lock();
entity_.update_ep_and_level(id, level);
spin_lock_.unlock();
mutex_.unlock();
}
return 0;
}
//! select_entry_point on hnsw level, ef = 1
void HnswRabitqAlgorithm::select_entry_point(level_t level,
node_id_t *entry_point,
ResultRecord *dist,
HnswRabitqContext *ctx) const {
auto &entity = ctx->get_entity();
HnswRabitqAddDistCalculator &dc = ctx->dist_calculator();
while (true) {
const Neighbors neighbors = entity.get_neighbors(level, *entry_point);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
uint32_t size = neighbors.size();
if (size == 0) {
break;
}
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
int ret = dc.get_vector(&neighbors[0], size, neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
if (ailego_unlikely(ret != 0)) {
break;
}
bool find_closer = false;
std::vector<float> dists(size);
std::vector<const void *> neighbor_vecs(size);
for (uint32_t i = 0; i < size; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
}
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
for (uint32_t i = 0; i < size; ++i) {
ResultRecord cur_dist = dists[i];
if (cur_dist < *dist) {
*entry_point = neighbors[i];
*dist = cur_dist;
find_closer = true;
}
}
if (!find_closer) {
break;
}
}
return;
}
void HnswRabitqAlgorithm::add_neighbors(node_id_t id, level_t level,
TopkHeap &topk_heap,
HnswRabitqContext *ctx) {
if (ailego_unlikely(topk_heap.size() == 0)) {
return;
}
HnswRabitqAddDistCalculator &dc = ctx->dist_calculator();
update_neighbors(dc, id, level, topk_heap);
// reverse update neighbors
for (size_t i = 0; i < topk_heap.size(); ++i) {
reverse_update_neighbors(dc, topk_heap[i].first, level, id,
topk_heap[i].second, ctx->update_heap());
}
return;
}
void HnswRabitqAlgorithm::search_neighbors(level_t level,
node_id_t *entry_point,
ResultRecord *dist, TopkHeap &topk,
HnswRabitqContext *ctx) const {
const auto &entity = ctx->get_entity();
HnswRabitqAddDistCalculator &dc = ctx->dist_calculator();
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (ctx->filter().is_valid()) {
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
}
candidates.clear();
visit.clear();
visit.set_visited(*entry_point);
if (!filter(*entry_point)) {
topk.emplace(*entry_point, *dist);
}
candidates.emplace(*entry_point, *dist);
while (!candidates.empty() && !ctx->reach_scan_limit()) {
auto top = candidates.begin();
node_id_t main_node = top->first;
ResultRecord main_dist = top->second;
if (topk.full() && main_dist > topk[0].second) {
break;
}
candidates.pop();
const Neighbors neighbors = entity.get_neighbors(level, main_node);
ailego_prefetch(neighbors.data);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) {
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_visit_dup_cnt())++;
}
continue;
}
visit.set_visited(node);
neighbor_ids[size++] = node;
}
if (size == 0) {
continue;
}
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
int ret = dc.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
if (ailego_unlikely(ret != 0)) {
break;
}
// do prefetch
static constexpr node_id_t BATCH_SIZE = 12;
static constexpr node_id_t PREFETCH_STEP = 2;
for (uint32_t i = 0; i < std::min(BATCH_SIZE * PREFETCH_STEP, size); ++i) {
ailego_prefetch(neighbor_vec_blocks[i].data());
}
// done
std::vector<float> dists(size);
std::vector<const void *> neighbor_vecs(size);
for (uint32_t i = 0; i < size; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
}
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
for (uint32_t i = 0; i < size; ++i) {
node_id_t node = neighbor_ids[i];
ResultRecord cur_dist = dists[i];
if ((!topk.full()) || cur_dist < topk[0].second) {
candidates.emplace(node, cur_dist);
// update entry_point for next level scan
if (cur_dist < *dist) {
*entry_point = node;
*dist = cur_dist;
}
if (!filter(node)) {
topk.emplace(node, cur_dist);
}
} // end if
} // end for
} // while
return;
}
void HnswRabitqAlgorithm::update_neighbors(HnswRabitqAddDistCalculator &dc,
node_id_t id, level_t level,
TopkHeap &topk_heap) {
topk_heap.sort();
uint32_t max_neighbor_cnt = entity_.neighbor_cnt(level);
if (topk_heap.size() <= static_cast<size_t>(entity_.prune_cnt())) {
if (topk_heap.size() <= static_cast<size_t>(max_neighbor_cnt)) {
entity_.update_neighbors(level, id, topk_heap);
return;
}
}
uint32_t cur_size = 0;
for (size_t i = 0; i < topk_heap.size(); ++i) {
node_id_t cur_node = topk_heap[i].first;
ResultRecord cur_node_dist = topk_heap[i].second;
bool good = true;
for (uint32_t j = 0; j < cur_size; ++j) {
ResultRecord tmp_dist = dc.dist(cur_node, topk_heap[j].first);
if (tmp_dist <= cur_node_dist) {
good = false;
break;
}
}
if (good) {
topk_heap[cur_size].first = cur_node;
topk_heap[cur_size].second = cur_node_dist;
cur_size++;
if (cur_size >= max_neighbor_cnt) {
break;
}
}
}
// when after-prune neighbor count is too seldom,
// we use this strategy to make-up enough edges
// not only just make-up out-degrees
// we also make-up enough in-degrees
uint32_t min_neighbors = entity_.min_neighbor_cnt();
for (size_t k = cur_size; cur_size < min_neighbors && k < topk_heap.size();
++k) {
bool exist = false;
for (size_t j = 0; j < cur_size; ++j) {
if (topk_heap[j].first == topk_heap[k].first) {
exist = true;
break;
}
}
if (!exist) {
topk_heap[cur_size].first = topk_heap[k].first;
topk_heap[cur_size].second = topk_heap[k].second;
cur_size++;
}
}
topk_heap.resize(cur_size);
entity_.update_neighbors(level, id, topk_heap);
return;
}
void HnswRabitqAlgorithm::reverse_update_neighbors(
HnswRabitqAddDistCalculator &dc, node_id_t id, level_t level,
node_id_t link_id, ResultRecord dist, TopkHeap &update_heap) {
const size_t max_neighbor_cnt = entity_.neighbor_cnt(level);
uint32_t lock_idx = id & kLockMask;
lock_pool_[lock_idx].lock();
const Neighbors neighbors = entity_.get_neighbors(level, id);
size_t size = neighbors.size();
ailego_assert_with(size <= max_neighbor_cnt, "invalid neighbor size");
if (size < max_neighbor_cnt) {
entity_.add_neighbor(level, id, size, link_id);
lock_pool_[lock_idx].unlock();
return;
}
update_heap.emplace(link_id, dist);
for (size_t i = 0; i < size; ++i) {
node_id_t node = neighbors[i];
ResultRecord cur_dist = dc.dist(id, node);
update_heap.emplace(node, cur_dist);
}
//! TODO: optimize prune
//! prune edges
update_heap.sort();
size_t cur_size = 0;
for (size_t i = 0; i < update_heap.size(); ++i) {
node_id_t cur_node = update_heap[i].first;
ResultRecord cur_node_dist = update_heap[i].second;
bool good = true;
for (size_t j = 0; j < cur_size; ++j) {
ResultRecord tmp_dist = dc.dist(cur_node, update_heap[j].first);
if (tmp_dist <= cur_node_dist) {
good = false;
break;
}
}
if (good) {
update_heap[cur_size].first = cur_node;
update_heap[cur_size].second = cur_node_dist;
cur_size++;
if (cur_size >= max_neighbor_cnt) {
break;
}
}
}
update_heap.resize(cur_size);
entity_.update_neighbors(level, id, update_heap);
lock_pool_[lock_idx].unlock();
update_heap.clear();
return;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,124 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
#include <ailego/parallel/lock.h>
#include "hnsw_rabitq_context.h"
#include "hnsw_rabitq_dist_calculator.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
//! hnsw graph algorithm implement
class HnswRabitqAlgorithm {
public:
typedef std::unique_ptr<HnswRabitqAlgorithm> UPointer;
public:
//! Constructor
explicit HnswRabitqAlgorithm(HnswRabitqEntity &entity);
//! Destructor
~HnswRabitqAlgorithm() = default;
//! Cleanup HnswRabitqAlgorithm
int cleanup();
//! Add a node to hnsw graph
//! @id: the node unique id
//! @level: a node will be add to graph in each level [0, level]
//! return 0 on success, or errCode in failure
int add_node(node_id_t id, level_t level, HnswRabitqContext *ctx);
//! Initiate HnswRabitqAlgorithm
int init() {
level_probas_.clear();
double level_mult =
1 / std::log(static_cast<double>(entity_.scaling_factor()));
for (int level = 0;; level++) {
// refers faiss get_random_level alg
double proba =
std::exp(-level / level_mult) * (1 - std::exp(-1 / level_mult));
if (proba < 1e-9) {
break;
}
level_probas_.push_back(proba);
}
return 0;
}
//! Generate a random level
//! return graph level
uint32_t get_random_level() const {
// gen rand float (0, 1)
double f = mt_() / static_cast<float>(mt_.max());
for (size_t level = 0; level < level_probas_.size(); level++) {
if (f < level_probas_[level]) {
return level;
}
f -= level_probas_[level];
}
return level_probas_.size() - 1;
}
private:
//! Select in upper layer to get entry point for next layer search
void select_entry_point(level_t level, node_id_t *entry_point,
ResultRecord *dist, HnswRabitqContext *ctx) const;
//! update node id neighbors from topkHeap, and reverse link is also updated
void add_neighbors(node_id_t id, level_t level, TopkHeap &topk_heap,
HnswRabitqContext *ctx);
//! Given a node id and level, search the nearest neighbors in graph
//! Note: the nearest neighbors result keeps in topk, and entry_point and
//! dist will be updated to current level nearest node id and distance
void search_neighbors(level_t level, node_id_t *entry_point,
ResultRecord *dist, TopkHeap &topk,
HnswRabitqContext *ctx) const;
//! Update the node's neighbors
void update_neighbors(HnswRabitqAddDistCalculator &dc, node_id_t id,
level_t level, TopkHeap &topk_heap);
//! Checking linkId could be id's new neighbor, and add as neighbor if true
//! @dc distance calculator
//! @updateHeap temporary heap in updating neighbors
void reverse_update_neighbors(HnswRabitqAddDistCalculator &dc, node_id_t id,
level_t level, node_id_t link_id,
ResultRecord dist, TopkHeap &update_heap);
private:
HnswRabitqAlgorithm(const HnswRabitqAlgorithm &) = delete;
HnswRabitqAlgorithm &operator=(const HnswRabitqAlgorithm &) = delete;
private:
static constexpr uint32_t kLockCnt{1U << 8};
static constexpr uint32_t kLockMask{kLockCnt - 1U};
HnswRabitqEntity &entity_;
mutable std::mt19937 mt_{};
std::vector<double> level_probas_{};
mutable ailego::SpinMutex spin_lock_{}; // global spin lock
std::mutex mutex_{}; // global mutex
// TODO: spin lock?
std::vector<std::mutex> lock_pool_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,554 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_builder.h"
#include <cstdlib>
#include <iostream>
#include <thread>
#include <ailego/pattern/defer.h>
#include <zvec/ailego/container/params.h>
#include <zvec/ailego/utility/string_helper.h>
#include <zvec/ailego/utility/time_helper.h>
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_factory.h"
#include "zvec/core/framework/index_logger.h"
#include "zvec/core/framework/index_memory.h"
#include "zvec/core/framework/index_meta.h"
#include "zvec/core/framework/index_provider.h"
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_entity.h"
#include "hnsw_rabitq_params.h"
#include "rabitq_converter.h"
#include "rabitq_params.h"
#include "rabitq_reformer.h"
namespace zvec {
namespace core {
HnswRabitqBuilder::HnswRabitqBuilder() {}
int HnswRabitqBuilder::init(const IndexMeta &meta,
const ailego::Params &params) {
LOG_INFO("Begin HnswRabitqBuilder::init");
meta_ = meta;
auto params_copy = params;
meta_.set_builder("HnswRabitqBuilder", HnswRabitqEntity::kRevision,
std::move(params_copy));
size_t memory_quota = 0UL;
params.get(PARAM_HNSW_RABITQ_BUILDER_MEMORY_QUOTA, &memory_quota);
params.get(PARAM_HNSW_RABITQ_BUILDER_THREAD_COUNT, &thread_cnt_);
params.get(PARAM_HNSW_RABITQ_BUILDER_MIN_NEIGHBOR_COUNT, &min_neighbor_cnt_);
params.get(PARAM_HNSW_RABITQ_BUILDER_EFCONSTRUCTION, &ef_construction_);
params.get(PARAM_HNSW_RABITQ_BUILDER_CHECK_INTERVAL_SECS,
&check_interval_secs_);
params.get(PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT,
&upper_max_neighbor_cnt_);
float multiplier = HnswRabitqEntity::kDefaultL0MaxNeighborCntMultiplier;
params.get(PARAM_HNSW_RABITQ_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER,
&multiplier);
l0_max_neighbor_cnt_ = multiplier * upper_max_neighbor_cnt_;
scaling_factor_ = upper_max_neighbor_cnt_;
params.get(PARAM_HNSW_RABITQ_BUILDER_SCALING_FACTOR, &scaling_factor_);
multiplier = HnswRabitqEntity::kDefaultNeighborPruneMultiplier;
params.get(PARAM_HNSW_RABITQ_BUILDER_NEIGHBOR_PRUNE_MULTIPLIER, &multiplier);
size_t prune_cnt = multiplier * upper_max_neighbor_cnt_;
if (ef_construction_ == 0) {
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
}
if (upper_max_neighbor_cnt_ == 0) {
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
}
if (upper_max_neighbor_cnt_ > kMaxNeighborCnt) {
LOG_ERROR("[%s] must be in range (0,%d]",
PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT.c_str(),
kMaxNeighborCnt);
return IndexError_InvalidArgument;
}
if (min_neighbor_cnt_ > upper_max_neighbor_cnt_) {
LOG_ERROR("[%s]-[%d] must be <= [%s]-[%d]",
PARAM_HNSW_RABITQ_BUILDER_MIN_NEIGHBOR_COUNT.c_str(),
min_neighbor_cnt_,
PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT.c_str(),
upper_max_neighbor_cnt_);
return IndexError_InvalidArgument;
}
if (l0_max_neighbor_cnt_ == 0) {
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
}
if (l0_max_neighbor_cnt_ > HnswRabitqEntity::kMaxNeighborCnt) {
LOG_ERROR("L0MaxNeighborCnt must be in range (0,%d)",
HnswRabitqEntity::kMaxNeighborCnt);
return IndexError_InvalidArgument;
}
if (scaling_factor_ == 0U) {
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
}
if (scaling_factor_ < 5 || scaling_factor_ > 1000) {
LOG_ERROR("[%s] must be in range [5,1000]",
PARAM_HNSW_RABITQ_BUILDER_SCALING_FACTOR.c_str());
return IndexError_InvalidArgument;
}
if (thread_cnt_ == 0) {
thread_cnt_ = std::thread::hardware_concurrency();
}
if (thread_cnt_ > std::thread::hardware_concurrency()) {
LOG_WARN("[%s] greater than cpu cores %zu",
PARAM_HNSW_RABITQ_BUILDER_THREAD_COUNT.c_str(),
static_cast<size_t>(std::thread::hardware_concurrency()));
}
if (prune_cnt == 0UL) {
prune_cnt = upper_max_neighbor_cnt_;
}
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!metric_) {
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
return IndexError_NoExist;
}
int ret = metric_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
return ret;
}
uint32_t total_bits = 0;
params.get(PARAM_RABITQ_TOTAL_BITS, &total_bits);
if (total_bits == 0) {
total_bits = kDefaultRabitqTotalBits;
}
if (total_bits < 1 || total_bits > 9) {
LOG_ERROR("Invalid total_bits: %zu, must be in [1, 9]", (size_t)total_bits);
return IndexError_InvalidArgument;
}
uint8_t ex_bits = total_bits - 1;
entity_.set_ex_bits(ex_bits);
uint32_t dimension = 0;
params.get(PARAM_HNSW_RABITQ_GENERAL_DIMENSION, &dimension);
if (dimension == 0) {
LOG_ERROR("%s not set", PARAM_HNSW_RABITQ_GENERAL_DIMENSION.c_str());
return IndexError_InvalidArgument;
}
if (dimension < kMinRabitqDimSize || dimension > kMaxRabitqDimSize) {
LOG_ERROR("Invalid dimension: %u, must be in [%d, %d]", dimension,
kMinRabitqDimSize, kMaxRabitqDimSize);
return IndexError_InvalidArgument;
}
entity_.update_rabitq_params_and_vector_size(dimension);
entity_.set_ef_construction(ef_construction_);
entity_.set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
entity_.set_min_neighbor_cnt(min_neighbor_cnt_);
entity_.set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
entity_.set_scaling_factor(scaling_factor_);
entity_.set_memory_quota(memory_quota);
entity_.set_prune_cnt(prune_cnt);
ret = entity_.init();
if (ret != 0) {
return ret;
}
alg_ = HnswRabitqAlgorithm::UPointer(new HnswRabitqAlgorithm(entity_));
ret = alg_->init();
if (ret != 0) {
return ret;
}
// Create and initialize RaBitQ converter
converter_ = std::make_shared<RabitqConverter>();
IndexMeta converter_meta = meta_;
converter_meta.set_dimension(dimension);
ret = converter_->init(converter_meta, params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqConverter: %d", ret);
return ret;
}
state_ = BUILD_STATE_INITED;
LOG_INFO(
"End HnswRabitqBuilder::init, params: rawVectorSize=%u vectorSize=%zu "
"efConstruction=%u "
"l0NeighborCnt=%u upperNeighborCnt=%u scalingFactor=%u "
"memoryQuota=%zu neighborPruneCnt=%zu metricName=%s ",
meta_.element_size(), entity_.vector_size(), ef_construction_,
l0_max_neighbor_cnt_, upper_max_neighbor_cnt_, scaling_factor_,
memory_quota, prune_cnt, meta_.metric_name().c_str());
return 0;
}
int HnswRabitqBuilder::cleanup(void) {
LOG_INFO("Begin HnswRabitqBuilder::cleanup");
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultL0MaxNeighborCnt;
min_neighbor_cnt_ = 0;
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
check_interval_secs_ = kDefaultLogIntervalSecs;
errcode_ = 0;
error_ = false;
entity_.cleanup();
if (alg_) {
alg_->cleanup();
}
meta_.clear();
metric_.reset();
stats_.clear_attributes();
stats_.set_trained_count(0UL);
stats_.set_built_count(0UL);
stats_.set_dumped_count(0UL);
stats_.set_discarded_count(0UL);
stats_.set_trained_costtime(0UL);
stats_.set_built_costtime(0UL);
stats_.set_dumped_costtime(0UL);
state_ = BUILD_STATE_INIT;
LOG_INFO("End HnswRabitqBuilder::cleanup");
return 0;
}
int HnswRabitqBuilder::train(IndexThreads::Pointer,
IndexHolder::Pointer holder) {
if (state_ != BUILD_STATE_INITED) {
LOG_ERROR("Init the builder before HnswRabitqBuilder::train");
return IndexError_NoReady;
}
if (!holder) {
LOG_ERROR("Input holder is nullptr while training index");
return IndexError_InvalidArgument;
}
if (!holder->is_matched(meta_)) {
LOG_ERROR("Input holder doesn't match index meta while training index");
return IndexError_Mismatch;
}
LOG_INFO("Begin HnswRabitqBuilder::train");
size_t trained_cost_time = 0;
size_t trained_count = 0;
int ret = train_converter_and_load_reformer(holder);
if (ret != 0) {
return ret;
}
if (metric_->support_train()) {
auto start_time = ailego::Monotime::MilliSeconds();
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator for holder failed");
return IndexError_Runtime;
}
while (iter->is_valid()) {
ret = metric_->train(iter->data(), meta_.dimension());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw build measure train failed, ret=%d", ret);
return ret;
}
iter->next();
++trained_count;
}
trained_cost_time = ailego::Monotime::MilliSeconds() - start_time;
}
stats_.set_trained_count(trained_count);
stats_.set_trained_costtime(trained_cost_time);
state_ = BUILD_STATE_TRAINED;
LOG_INFO("End HnswRabitqBuilder::train");
return 0;
}
int HnswRabitqBuilder::train_converter_and_load_reformer(
IndexHolder::Pointer holder) {
// Train converter (KMeans clustering)
int ret = converter_->train(holder);
if (ret != 0) {
LOG_ERROR("Failed to train RabitqConverter: %d", ret);
return ret;
}
auto memory_dumper = IndexFactory::CreateDumper("MemoryDumper");
memory_dumper->init(ailego::Params());
std::string file_id = ailego::StringHelper::Concat(
"rabitq_converter_", ailego::Monotime::MilliSeconds(), rand());
ret = memory_dumper->create(file_id);
if (ret != 0) {
LOG_ERROR("Failed to create memory dumper: %d", ret);
return ret;
}
// Release memory
AILEGO_DEFER([&file_id]() { IndexMemory::Instance()->remove(file_id); });
ret = converter_->dump(memory_dumper);
if (ret != 0) {
LOG_ERROR("Failed to dump RabitqConverter: %d", ret);
return ret;
}
ret = memory_dumper->close();
if (ret != 0) {
LOG_ERROR("Failed to close memory dumper: %d", ret);
return ret;
}
reformer_ = std::make_shared<RabitqReformer>();
ailego::Params reformer_params;
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
ret = reformer_->init(reformer_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
return ret;
}
auto memory_storage = IndexFactory::CreateStorage("MemoryReadStorage");
ret = memory_storage->open(file_id, false);
if (ret != 0) {
LOG_ERROR("Failed to open memory storage: %d", ret);
return ret;
}
ret = reformer_->load(memory_storage);
if (ret != 0) {
LOG_ERROR("Failed to load RabitqReformer: %d", ret);
return ret;
}
return 0;
}
int HnswRabitqBuilder::train(const IndexTrainer::Pointer & /*trainer*/) {
if (state_ != BUILD_STATE_INITED) {
LOG_ERROR("Init the builder before HnswRabitqBuilder::train");
return IndexError_NoReady;
}
LOG_INFO("Begin HnswRabitqBuilder::train by trainer");
stats_.set_trained_count(0UL);
stats_.set_trained_costtime(0UL);
state_ = BUILD_STATE_TRAINED;
LOG_INFO("End HnswRabitqBuilder::train by trainer");
return 0;
}
int HnswRabitqBuilder::build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) {
if (state_ != BUILD_STATE_TRAINED) {
LOG_ERROR("Train the index before HnswRabitqBuilder::build");
return IndexError_NoReady;
}
if (!holder) {
LOG_ERROR("Input holder is nullptr while building index");
return IndexError_InvalidArgument;
}
if (!holder->is_matched(meta_)) {
LOG_ERROR("Input holder doesn't match index meta while building index");
return IndexError_Mismatch;
}
IndexProvider::Pointer provider =
std::dynamic_pointer_cast<IndexProvider>(holder);
if (!provider) {
LOG_ERROR("Rabitq builder expect IndexProvider");
return IndexError_InvalidArgument;
}
if (!threads) {
threads = std::make_shared<SingleQueueIndexThreads>(thread_cnt_, false);
}
auto start_time = ailego::Monotime::MilliSeconds();
LOG_INFO("Begin HnswRabitqBuilder::build");
if (holder->count() != static_cast<size_t>(-1)) {
LOG_DEBUG("HnswRabitqBuilder holder documents count %lu", holder->count());
int ret = entity_.reserve_space(holder->count());
if (ret != 0) {
LOG_ERROR("HnswBuilde reserver space failed");
return ret;
}
}
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator for holder failed");
return IndexError_Runtime;
}
int ret;
error_ = false;
IndexQueryMeta ometa;
ometa.set_meta(holder->data_type(), holder->dimension());
while (iter->is_valid()) {
const void *vec = iter->data();
// quantize vector
std::string converted_vector;
IndexQueryMeta converted_meta;
ret = reformer_->convert(vec, ometa, &converted_vector, &converted_meta);
if (ret != 0) {
LOG_ERROR("Rabitq hnsw convert failed, ret=%d", ret);
return ret;
}
level_t level = alg_->get_random_level();
node_id_t id;
if (converted_vector.size() != entity_.vector_size()) {
LOG_ERROR(
"Converted vector size %zu is not equal to entity vector size %zu",
converted_vector.size(), entity_.vector_size());
return IndexError_InvalidArgument;
}
ret = entity_.add_vector(level, iter->key(), converted_vector.data(), &id);
if (ailego_unlikely(ret != 0)) {
return ret;
}
iter->next();
}
LOG_INFO("Finished save vector, start build graph...");
auto task_group = threads->make_group();
if (!task_group) {
LOG_ERROR("Failed to create task group");
return IndexError_Runtime;
}
std::atomic<node_id_t> finished{0};
for (size_t i = 0; i < threads->count(); ++i) {
task_group->submit(ailego::Closure ::New(this, &HnswRabitqBuilder::do_build,
i, threads->count(), provider,
&finished));
}
while (!task_group->is_finished()) {
std::unique_lock<std::mutex> lk(mutex_);
cond_.wait_until(lk, std::chrono::system_clock::now() +
std::chrono::seconds(check_interval_secs_));
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to build index while waiting finish");
return errcode_;
}
LOG_INFO("Built cnt %zu, finished percent %.3f%%",
static_cast<size_t>(finished.load()),
finished.load() * 100.0f / entity_.doc_cnt());
}
if (error_.load(std::memory_order_acquire)) {
LOG_ERROR("Failed to build index while waiting finish");
return errcode_;
}
task_group->wait_finish();
stats_.set_built_count(finished.load());
stats_.set_built_costtime(ailego::Monotime::MilliSeconds() - start_time);
state_ = BUILD_STATE_BUILT;
LOG_INFO("End HnswRabitqBuilder::build with RaBitQ quantization");
return 0;
}
void HnswRabitqBuilder::do_build(node_id_t idx, size_t step_size,
IndexProvider::Pointer provider,
std::atomic<node_id_t> *finished) {
AILEGO_DEFER([&]() {
std::lock_guard<std::mutex> latch(mutex_);
cond_.notify_one();
});
HnswRabitqContext *ctx = new (std::nothrow) HnswRabitqContext(
meta_.dimension(), metric_,
std::shared_ptr<HnswRabitqEntity>(&entity_, [](HnswRabitqEntity *) {}));
if (ailego_unlikely(ctx == nullptr)) {
if (!error_.exchange(true)) {
LOG_ERROR("Failed to create context");
errcode_ = IndexError_NoMemory;
}
return;
}
HnswRabitqContext::Pointer auto_ptr(ctx);
ctx->set_provider(std::move(provider));
ctx->set_max_scan_num(entity_.doc_cnt());
int ret = ctx->init(HnswRabitqContext::kBuilderContext);
if (ret != 0) {
if (!error_.exchange(true)) {
LOG_ERROR("Failed to init context");
errcode_ = IndexError_Runtime;
}
return;
}
for (node_id_t id = idx; id < entity_.doc_cnt(); id += step_size) {
ctx->reset_query(ctx->dist_calculator().get_vector(id));
ret = alg_->add_node(id, entity_.get_level(id), ctx);
if (ailego_unlikely(ret != 0)) {
if (!error_.exchange(true)) {
LOG_ERROR("Hnsw graph add node failed");
errcode_ = ret;
}
return;
}
ctx->clear();
(*finished)++;
}
}
int HnswRabitqBuilder::dump(const IndexDumper::Pointer &dumper) {
if (state_ != BUILD_STATE_BUILT) {
LOG_INFO("Build the index before HnswRabitqBuilder::dump");
return IndexError_NoReady;
}
LOG_INFO("Begin HnswRabitqBuilder::dump");
meta_.set_searcher("HnswRabitqSearcher", HnswRabitqEntity::kRevision,
ailego::Params());
auto start_time = ailego::Monotime::MilliSeconds();
int ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
if (ret != 0) {
LOG_ERROR("Failed to serialize meta into dumper.");
return ret;
}
// Dump RaBitQ centroids first
if (converter_) {
ret = converter_->dump(dumper);
if (ret != 0) {
LOG_ERROR("Failed to dump RabitqConverter: %d", ret);
return ret;
}
LOG_INFO("RaBitQ centroids dumped: %zu bytes, cost %zu ms",
converter_->stats().dumped_size(),
static_cast<size_t>(converter_->stats().dumped_costtime()));
}
ret = entity_.dump(dumper);
if (ret != 0) {
LOG_ERROR("HnswRabitqBuilder dump index failed");
return ret;
}
stats_.set_dumped_count(entity_.doc_cnt());
stats_.set_dumped_costtime(ailego::Monotime::MilliSeconds() - start_time);
LOG_INFO("End HnswRabitqBuilder::dump");
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,101 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/ailego/parallel/thread_pool.h>
#include "zvec/core/framework/index_builder.h"
#include "zvec/core/framework/index_converter.h"
#include "zvec/core/framework/index_reformer.h"
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_builder_entity.h"
namespace zvec {
namespace core {
class HnswRabitqBuilder : public IndexBuilder {
public:
//! Constructor
HnswRabitqBuilder();
//! Initialize the builder
virtual int init(const IndexMeta &meta,
const ailego::Params &params) override;
//! Cleanup the builder
virtual int cleanup(void) override;
//! Train the data
virtual int train(IndexThreads::Pointer,
IndexHolder::Pointer holder) override;
//! Train the data
virtual int train(const IndexTrainer::Pointer &trainer) override;
//! Build the index
virtual int build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
//! Dump index into storage
virtual int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
return stats_;
}
private:
void do_build(node_id_t idx, size_t step_size,
IndexProvider::Pointer provider,
std::atomic<node_id_t> *finished);
int train_converter_and_load_reformer(IndexHolder::Pointer holder);
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
constexpr static uint32_t kMaxNeighborCnt = 65535;
private:
enum BUILD_STATE {
BUILD_STATE_INIT = 0,
BUILD_STATE_INITED = 1,
BUILD_STATE_TRAINED = 2,
BUILD_STATE_BUILT = 3
};
HnswRabitqBuilderEntity entity_{};
HnswRabitqAlgorithm::UPointer alg_; // impl graph algorithm
uint32_t thread_cnt_{0};
uint32_t min_neighbor_cnt_{0};
uint32_t upper_max_neighbor_cnt_{
HnswRabitqEntity::kDefaultUpperMaxNeighborCnt};
uint32_t l0_max_neighbor_cnt_{HnswRabitqEntity::kDefaultL0MaxNeighborCnt};
uint32_t ef_construction_{HnswRabitqEntity::kDefaultEfConstruction};
uint32_t scaling_factor_{HnswRabitqEntity::kDefaultScalingFactor};
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
int errcode_{0};
std::atomic_bool error_{false};
IndexMeta meta_{};
IndexMetric::Pointer metric_{};
IndexConverter::Pointer converter_{}; // RaBitQ converter
IndexReformer::Pointer reformer_{}; // RaBitQ reformer
std::mutex mutex_{};
std::condition_variable cond_{};
Stats stats_{};
BUILD_STATE state_{BUILD_STATE_INIT};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,199 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_builder_entity.h"
#include <iostream>
#include <zvec/ailego/hash/crc32c.h>
#include "utility/sparse_utility.h"
namespace zvec {
namespace core {
HnswRabitqBuilderEntity::HnswRabitqBuilderEntity() {
update_ep_and_level(kInvalidNodeId, 0U);
}
int HnswRabitqBuilderEntity::cleanup() {
memory_quota_ = 0UL;
neighbors_size_ = 0U;
upper_neighbors_size_ = 0U;
padding_size_ = 0U;
vectors_buffer_.clear();
keys_buffer_.clear();
neighbors_buffer_.clear();
upper_neighbors_buffer_.clear();
neighbors_index_.clear();
vectors_buffer_.shrink_to_fit();
keys_buffer_.shrink_to_fit();
neighbors_buffer_.shrink_to_fit();
upper_neighbors_buffer_.shrink_to_fit();
neighbors_index_.shrink_to_fit();
this->HnswRabitqEntity::cleanup();
return 0;
}
int HnswRabitqBuilderEntity::init() {
size_t size = vector_size();
//! aligned size to 32
set_node_size(AlignSize(size));
//! if node size is aligned to 1k, the build performance will downgrade
if (node_size() % 1024 == 0) {
set_node_size(AlignSize(node_size() + 1));
}
padding_size_ = node_size() - size;
neighbors_size_ = neighbors_size();
upper_neighbors_size_ = upper_neighbors_size();
return 0;
}
int HnswRabitqBuilderEntity::reserve_space(size_t docs) {
if (memory_quota_ > 0 && (node_size() * docs + neighbors_size_ * docs +
sizeof(NeighborIndex) * docs >
memory_quota_)) {
return IndexError_NoMemory;
}
vectors_buffer_.reserve(node_size() * docs);
keys_buffer_.reserve(sizeof(key_t) * docs);
neighbors_buffer_.reserve(neighbors_size_ * docs);
neighbors_index_.reserve(docs);
return 0;
}
int HnswRabitqBuilderEntity::add_vector(level_t level, key_t key,
const void *vec, node_id_t *id) {
if (memory_quota_ > 0 &&
(vectors_buffer_.capacity() + keys_buffer_.capacity() +
neighbors_buffer_.capacity() + upper_neighbors_buffer_.capacity() +
neighbors_index_.capacity() * sizeof(NeighborIndex)) > memory_quota_) {
LOG_ERROR("Add vector failed, used memory exceed quota, cur_doc=%zu",
static_cast<size_t>(doc_cnt()));
return IndexError_NoMemory;
}
vectors_buffer_.append(reinterpret_cast<const char *>(vec), vector_size());
vectors_buffer_.append(padding_size_, '\0');
keys_buffer_.append(reinterpret_cast<const char *>(&key), sizeof(key));
// init level 0 neighbors
neighbors_buffer_.append(neighbors_size_, '\0');
neighbors_index_.emplace_back(upper_neighbors_buffer_.size(), level);
// init upper layer neighbors
for (level_t cur_level = 1; cur_level <= level; ++cur_level) {
upper_neighbors_buffer_.append(upper_neighbors_size_, '\0');
}
*id = (*mutable_doc_cnt())++;
return 0;
}
key_t HnswRabitqBuilderEntity::get_key(node_id_t id) const {
return *(reinterpret_cast<const key_t *>(keys_buffer_.data() +
id * sizeof(key_t)));
}
const void *HnswRabitqBuilderEntity::get_vector(node_id_t id) const {
return vectors_buffer_.data() + id * node_size();
}
int HnswRabitqBuilderEntity::get_vector(
const node_id_t id, IndexStorage::MemoryBlock &block) const {
const void *vec = get_vector(id);
block.reset((void *)vec);
return 0;
}
int HnswRabitqBuilderEntity::get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const {
for (uint32_t i = 0; i < count; ++i) {
vecs[i] = vectors_buffer_.data() + ids[i] * node_size();
}
return 0;
}
int HnswRabitqBuilderEntity::get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
std::vector<const void *> vecs(count);
get_vector(ids, count, vecs.data());
for (uint32_t i = 0; i < count; ++i) {
vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i]));
}
return 0;
}
const Neighbors HnswRabitqBuilderEntity::get_neighbors(level_t level,
node_id_t id) const {
const NeighborsHeader *hd = get_neighbor_header(level, id);
return {hd->neighbor_cnt, hd->neighbors};
}
int HnswRabitqBuilderEntity::update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors) {
NeighborsHeader *hd =
const_cast<NeighborsHeader *>(get_neighbor_header(level, id));
for (size_t i = 0; i < neighbors.size(); ++i) {
hd->neighbors[i] = neighbors[i].first;
}
hd->neighbor_cnt = neighbors.size();
// std::cout << "id: " << id << ", neighbour, id: ";
// for (size_t i = 0; i < neighbors.size(); ++i) {
// if (i == neighbors.size()-1)
// std::cout << neighbors[i].first << ", score:" << neighbors[i].second <<
// std::endl;
// else
// std::cout << neighbors[i].first << ", score:" << neighbors[i].second <<
// ", id: ";
// }
return 0;
}
void HnswRabitqBuilderEntity::add_neighbor(level_t level, node_id_t id,
uint32_t /*size*/,
node_id_t neighbor_id) {
NeighborsHeader *hd =
const_cast<NeighborsHeader *>(get_neighbor_header(level, id));
hd->neighbors[hd->neighbor_cnt++] = neighbor_id;
return;
}
int HnswRabitqBuilderEntity::dump(const IndexDumper::Pointer &dumper) {
key_t *keys =
reinterpret_cast<key_t *>(const_cast<char *>(keys_buffer_.data()));
auto ret =
dump_segments(dumper, keys, [&](node_id_t id) { return get_level(id); });
if (ailego_unlikely(ret < 0)) {
return ret;
}
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,139 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/ailego/internal/platform.h>
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
class HnswRabitqBuilderEntity : public HnswRabitqEntity {
public:
//! Add vector and key to hnsw entity, and local id will be saved to id
virtual int add_vector(level_t level, key_t key, const void *vec,
node_id_t *id) override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
//! Get vector feature data by key
virtual const void *get_vector(node_id_t id) const override;
//! Batch get vectors feature data by keys
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
//! Get the node id's neighbors on graph level
const NeighborsHeader *get_neighbor_header(level_t level,
node_id_t id) const {
if (level == 0) {
return reinterpret_cast<const NeighborsHeader *>(
neighbors_buffer_.data() + neighbors_size_ * id);
} else {
size_t offset = neighbors_index_[id].offset;
return reinterpret_cast<const NeighborsHeader *>(
upper_neighbors_buffer_.data() + offset +
(level - 1) * upper_neighbors_size_);
}
}
//! Get the node id's neighbors on graph level
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
//! Replace node id in level's neighbors
virtual int update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors)
override;
//! add a neighbor to id in graph level
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
//! Dump the hnsw graph to dumper
virtual int dump(const IndexDumper::Pointer &dumper) override;
//! Cleanup the entity
virtual int cleanup(void) override;
public:
//! Constructor
HnswRabitqBuilderEntity();
//! Get the node graph level by id
level_t get_level(node_id_t id) const {
return neighbors_index_[id].level;
}
//! Init builerEntity
int init();
//! reserve buffer space for documents
//! @param docs number of documents
int reserve_space(size_t docs);
//! Set memory quota params
inline void set_memory_quota(size_t memory_quota) {
memory_quota_ = memory_quota;
}
//! Get neighbors size
inline size_t neighbors_size() const {
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
}
//! Get upper neighbors size
inline size_t upper_neighbors_size() const {
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
}
public:
HnswRabitqBuilderEntity(const HnswRabitqBuilderEntity &) = delete;
HnswRabitqBuilderEntity &operator=(const HnswRabitqBuilderEntity &) = delete;
private:
friend class HnswRabitqSearcherEntity;
//! class internal used only
struct NeighborIndex {
NeighborIndex(size_t off, level_t l) : offset(off), level(l) {}
uint64_t offset : 48;
uint64_t level : 16;
};
std::string vectors_buffer_{}; // aligned vectors
std::string keys_buffer_{}; // aligned vectors
std::string neighbors_buffer_{}; // level 0 neighbors buffer
std::string upper_neighbors_buffer_{}; // upper layer neighbors buffer
std::string sparse_data_buffer_{}; // aligned spase data buffer
size_t sparse_data_offset_{0}; //
// upper layer offset + level in upper_neighbors_buffer_
std::vector<NeighborIndex> neighbors_index_{};
size_t memory_quota_{0UL};
size_t neighbors_size_{0U}; // level 0 neighbors size
size_t upper_neighbors_size_{0U}; // level 0 neighbors size
size_t padding_size_{}; // padding size for each vector element
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,221 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_chunk.h"
#include <chrono>
#include <random>
#include <zvec/ailego/hash/crc32c.h>
#include <zvec/ailego/utility/time_helper.h>
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_helper.h"
#include "zvec/core/framework/index_logger.h"
#include "zvec/core/framework/index_streamer.h"
namespace zvec {
namespace core {
int HnswRabitqChunkBroker::init_storage(size_t chunk_size) {
chunk_meta_.clear();
chunk_meta_.chunk_size = chunk_size;
chunk_meta_.create_time = ailego::Realtime::Seconds();
stats_.set_create_time(chunk_meta_.create_time);
chunk_meta_.update_time = ailego::Realtime::Seconds();
stats_.set_update_time(chunk_meta_.update_time);
//! alloc meta chunk
size_t size = sizeof(HnswChunkMeta);
size = (size + page_mask_) & (~page_mask_);
const std::string segment_id =
make_segment_id(CHUNK_TYPE_META, kDefaultChunkSeqId);
int ret = stg_->append(segment_id, size);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Storage append segment failed for %s", IndexError::What(ret));
return ret;
}
chunk_meta_segment_ = get_chunk(CHUNK_TYPE_META, kDefaultChunkSeqId);
if (ailego_unlikely(!chunk_meta_segment_)) {
LOG_ERROR("Get meta segment failed");
return IndexError_Runtime;
}
//! update meta info and write to storage
chunk_meta_.chunk_cnts[CHUNK_TYPE_META] += 1;
chunk_meta_.total_size += size;
(*stats_.mutable_index_size()) += size;
size = chunk_meta_segment_->write(0UL, &chunk_meta_, sizeof(HnswChunkMeta));
if (ailego_unlikely(size != sizeof(HnswChunkMeta))) {
LOG_ERROR("Storage write data failed, wsize=%zu", size);
return IndexError_WriteData;
}
return 0;
}
int HnswRabitqChunkBroker::load_storage(size_t chunk_size) {
IndexStorage::MemoryBlock data_block;
size_t size = chunk_meta_segment_->read(0UL, data_block,
chunk_meta_segment_->data_size());
if (size != sizeof(HnswChunkMeta)) {
LOG_ERROR("Invalid hnsw meta chunk, read size=%zu chunk size=%zu", size,
chunk_meta_segment_->data_size());
return IndexError_InvalidFormat;
}
std::memcpy(&chunk_meta_, data_block.data(), size);
if (chunk_meta_.chunk_size != chunk_size) {
LOG_ERROR(
"Params hnsw chunk size=%zu mismatch from previous %zu "
"in index",
chunk_size, (size_t)chunk_meta_.chunk_size);
return IndexError_Mismatch;
}
*stats_.mutable_check_point() = stg_->check_point();
stats_.set_revision_id(chunk_meta_.revision_id);
stats_.set_update_time(chunk_meta_.update_time);
stats_.set_create_time(chunk_meta_.create_time);
char create_time[32];
char update_time[32];
ailego::Realtime::Gmtime(chunk_meta_.create_time, "%Y-%m-%d %H:%M:%S",
create_time, sizeof(create_time));
ailego::Realtime::Gmtime(chunk_meta_.update_time, "%Y-%m-%d %H:%M:%S",
update_time, sizeof(update_time));
LOG_DEBUG(
"Load index, indexSize=%zu chunkSize=%zu nodeChunks=%zu "
"upperNeighborChunks=%zu revisionId=%zu "
"createTime=%s updateTime=%s",
(size_t)chunk_meta_.total_size, (size_t)chunk_meta_.chunk_size,
(size_t)chunk_meta_.chunk_cnts[CHUNK_TYPE_NODE],
(size_t)chunk_meta_.chunk_cnts[CHUNK_TYPE_UPPER_NEIGHBOR],
(size_t)chunk_meta_.revision_id, create_time, update_time);
return 0;
}
int HnswRabitqChunkBroker::open(IndexStorage::Pointer stg,
size_t max_index_size, size_t chunk_size,
bool check_crc) {
if (ailego_unlikely(stg_)) {
LOG_ERROR("An storage instance is already opened");
return IndexError_Duplicate;
}
stg_ = std::move(stg);
if (stg_->isHugePage()) {
page_mask_ = ailego::MemoryHelper::HugePageSize() - 1;
} else {
page_mask_ = ailego::MemoryHelper::PageSize() - 1;
}
check_crc_ = check_crc;
max_chunks_size_ = max_index_size;
dirty_ = false;
const std::string segment_id =
make_segment_id(CHUNK_TYPE_META, kDefaultChunkSeqId);
chunk_meta_segment_ = stg_->get(segment_id);
if (!chunk_meta_segment_) {
LOG_DEBUG("Create new index");
return init_storage(chunk_size);
}
return load_storage(chunk_size);
}
int HnswRabitqChunkBroker::close(void) {
flush(0UL);
stg_.reset();
check_crc_ = false;
dirty_ = false;
return 0;
}
int HnswRabitqChunkBroker::flush(uint64_t checkpoint) {
ailego_assert_with(chunk_meta_segment_, "invalid meta segment");
chunk_meta_.update_time = ailego::Realtime::Seconds();
stats_.set_update_time(chunk_meta_.update_time);
size_t size =
chunk_meta_segment_->write(0UL, &chunk_meta_, sizeof(HnswChunkMeta));
if (ailego_unlikely(size != sizeof(HnswChunkMeta))) {
LOG_ERROR("Storage write data failed, wsize=%zu", size);
}
stg_->refresh(checkpoint);
int ret = stg_->flush();
if (ret == 0) {
(*stats_.mutable_check_point()) = checkpoint;
} else {
LOG_ERROR("Storage flush failed for %s", IndexError::What(ret));
}
return ret;
}
std::pair<int, Chunk::Pointer> HnswRabitqChunkBroker::alloc_chunk(
int type, uint64_t seq_id, size_t size) {
ailego_assert_with(type < CHUNK_TYPE_MAX, "chunk type overflow");
Chunk::Pointer chunk;
if (ailego_unlikely(!stg_)) {
LOG_ERROR("Init storage first");
return std::make_pair(IndexError_Uninitialized, chunk);
}
//! check exist a empty chunk with the same name
chunk = get_chunk(type, seq_id);
if (chunk) {
if (ailego_unlikely(chunk->capacity() == size &&
chunk->data_size() == 0UL)) {
LOG_ERROR("Exist invalid chunk size %zu, expect size %zu",
chunk->capacity(), size);
chunk.reset();
return std::make_pair(IndexError_Runtime, chunk);
}
return std::make_pair(0, chunk);
}
//! align to page size
size = (size + page_mask_) & (~page_mask_);
if (ailego_unlikely(chunk_meta_.total_size + size >= max_chunks_size_)) {
LOG_ERROR("No space to new a chunk, curIndexSize=%zu allocSize=%zu",
(size_t)chunk_meta_.total_size, size);
return std::make_pair(IndexError_IndexFull, chunk);
}
std::string segment_id = make_segment_id(type, seq_id);
int ret = stg_->append(segment_id, size);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Storage append segment failed for %s", IndexError::What(ret));
return std::make_pair(ret, chunk);
}
chunk_meta_.chunk_cnts[type] += 1;
chunk_meta_.total_size += size;
(*stats_.mutable_index_size()) += size;
size = chunk_meta_segment_->write(0UL, &chunk_meta_, sizeof(HnswChunkMeta));
if (ailego_unlikely(size != sizeof(HnswChunkMeta))) {
LOG_ERROR("Storage append segment failed, wsize=%zu", size);
}
chunk = get_chunk(type, seq_id);
return std::make_pair(chunk ? 0 : IndexError_NoMemory, chunk);
}
Chunk::Pointer HnswRabitqChunkBroker::get_chunk(int type,
uint64_t seq_id) const {
std::string segment_id = make_segment_id(type, seq_id);
return stg_->get(segment_id);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,140 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
#include <string.h>
#include <unistd.h>
#include <atomic>
#include <cstddef>
#include <mutex>
#include <ailego/utility/memory_helper.h>
#include <zvec/ailego/internal/platform.h>
#include <zvec/ailego/utility/string_helper.h>
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_logger.h"
#include "zvec/core/framework/index_storage.h"
#include "zvec/core/framework/index_streamer.h"
namespace zvec {
namespace core {
using Chunk = IndexStorage::Segment;
class HnswRabitqChunkBroker {
public:
typedef std::shared_ptr<HnswRabitqChunkBroker> Pointer;
enum CHUNK_TYPE {
CHUNK_TYPE_HEADER = 1,
CHUNK_TYPE_META = 2,
CHUNK_TYPE_NODE = 3,
CHUNK_TYPE_UPPER_NEIGHBOR = 4,
CHUNK_TYPE_NEIGHBOR_INDEX = 5,
CHUNK_TYPE_SPARSE_NODE = 6,
CHUNK_TYPE_MAX = 8
};
static constexpr size_t kDefaultChunkSeqId = 0UL;
HnswRabitqChunkBroker(IndexStreamer::Stats &stats) : stats_(stats) {}
//! Open storage
int open(IndexStorage::Pointer stg, size_t max_index_size, size_t chunk_size,
bool check_crc);
int close(void);
int flush(uint64_t checkpoint);
//! alloc a new chunk with size, not thread-safe
std::pair<int, Chunk::Pointer> alloc_chunk(int type, uint64_t seq_id,
size_t size);
//! alloc a new chunk with chunk size
inline std::pair<int, Chunk::Pointer> alloc_chunk(int type, uint64_t seq_id) {
return alloc_chunk(type, seq_id, chunk_meta_.chunk_size);
}
Chunk::Pointer get_chunk(int type, uint64_t seq_id) const;
inline size_t get_chunk_cnt(int type) const {
ailego_assert_with(type < CHUNK_TYPE_MAX, "chunk type overflow");
return chunk_meta_.chunk_cnts[type];
}
inline bool dirty(void) const {
return dirty_;
}
inline void mark_dirty(void) {
if (!dirty_) {
dirty_ = true;
chunk_meta_.revision_id += 1;
stats_.set_revision_id(chunk_meta_.revision_id);
}
}
const IndexStorage::Pointer storage(void) const {
return stg_;
}
private:
HnswRabitqChunkBroker(const HnswRabitqChunkBroker &) = delete;
HnswRabitqChunkBroker &operator=(const HnswRabitqChunkBroker &) = delete;
struct HnswChunkMeta {
HnswChunkMeta(void) {
memset(this, 0, sizeof(HnswChunkMeta));
}
void clear() {
memset(this, 0, sizeof(HnswChunkMeta));
}
uint64_t chunk_cnts[CHUNK_TYPE_MAX];
uint64_t chunk_size; // size of per chunk
uint64_t total_size; // total size of allocated chunk
uint64_t revision_id; // index revision
uint64_t create_time;
uint64_t update_time;
uint64_t reserved[3];
};
static_assert(sizeof(HnswChunkMeta) % 32 == 0,
"HnswChunkMeta must be aligned with 32 bytes");
//! Init the storage after open an empty index
int init_storage(size_t chunk_size);
//! Load index from storage
int load_storage(size_t chunk_size);
static inline const std::string make_segment_id(int type, uint64_t seq_id) {
return "HnswT" + ailego::StringHelper::ToString(type) + "S" +
ailego::StringHelper::ToString(seq_id);
}
private:
IndexStreamer::Stats &stats_;
HnswChunkMeta chunk_meta_{};
size_t page_mask_{0UL};
size_t max_chunks_size_{0UL};
IndexStorage::Pointer stg_{};
IndexStorage::Segment::Pointer chunk_meta_segment_{};
bool check_crc_{false};
bool dirty_{false}; // set as true if index is modified , the flag
// will not be cleared even if flushed
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,297 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_context.h"
#include <chrono>
#include "hnsw_rabitq_params.h"
namespace zvec {
namespace core {
HnswRabitqContext::HnswRabitqContext(size_t dimension,
const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity)
: IndexContext(metric),
entity_(entity),
add_dc_(entity_.get(), metric, dimension) {}
HnswRabitqContext::HnswRabitqContext(const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity)
: IndexContext(metric), entity_(entity), add_dc_(entity_.get(), metric) {}
HnswRabitqContext::~HnswRabitqContext() {
visit_filter_.destroy();
}
int HnswRabitqContext::init(ContextType type) {
int ret;
uint32_t doc_cnt;
type_ = type;
switch (type) {
case kBuilderContext:
ret = visit_filter_.init(VisitFilter::ByteMap, entity_->doc_cnt(),
max_scan_num_, negative_probability_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
candidates_.limit(max_scan_num_);
update_heap_.limit(entity_->l0_neighbor_cnt() + 1);
break;
case kSearcherContext:
ret = visit_filter_.init(filter_mode_, entity_->doc_cnt(), max_scan_num_,
negative_probability_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
candidates_.limit(max_scan_num_);
break;
case kStreamerContext:
// maxScanNum is unknown if inited from streamer, so the docCnt may
// change. we need to compute maxScanNum by scan ratio, and preserve
// max_doc_cnt space from visit filter
doc_cnt = entity_->doc_cnt();
max_scan_num_ = compute_max_scan_num(doc_cnt);
reserve_max_doc_cnt_ = doc_cnt + compute_reserve_cnt(doc_cnt);
ret = visit_filter_.init(filter_mode_, reserve_max_doc_cnt_,
max_scan_num_, negative_probability_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
update_heap_.limit(entity_->l0_neighbor_cnt() + 1);
candidates_.limit(max_scan_num_);
check_need_adjuct_ctx();
break;
default:
LOG_ERROR("Init context failed");
return IndexError_Runtime;
}
return 0;
}
int HnswRabitqContext::update(const ailego::Params &params) {
auto update_visit_filter_param = [&]() {
bool need_update = false;
std::string p;
switch (type_) {
case kSearcherContext:
p = PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_ENABLE;
break;
case kStreamerContext:
p = PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_ENABLE;
break;
}
if (params.has(p)) {
bool bf_enabled;
params.get(p, &bf_enabled);
if (bf_enabled ^ (filter_mode_ == VisitFilter::BloomFilter)) {
need_update = true;
filter_mode_ =
bf_enabled ? VisitFilter::BloomFilter : VisitFilter::ByteMap;
}
}
float prob = negative_probability_;
p.clear();
switch (type_) {
case kSearcherContext:
p = PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB;
break;
case kStreamerContext:
p = PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB;
break;
}
params.get(p, &prob);
if (filter_mode_ == VisitFilter::BloomFilter &&
std::abs(prob - negative_probability_) > 1e-6) {
need_update = true;
}
if (need_update) {
visit_filter_.destroy();
int max_doc_cnt = 0;
if (type_ == kSearcherContext) {
max_doc_cnt = entity_->doc_cnt();
} else {
max_doc_cnt = reserve_max_doc_cnt_;
}
int ret = visit_filter_.init(filter_mode_, max_doc_cnt, max_scan_num_,
negative_probability_);
if (ret != 0) {
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
return ret;
}
}
return 0;
};
switch (type_) {
case kSearcherContext:
if (params.has(PARAM_HNSW_RABITQ_SEARCHER_EF)) {
params.get(PARAM_HNSW_RABITQ_SEARCHER_EF, &ef_);
topk_heap_.limit(std::max(topk_, ef_));
}
if (params.has(PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO)) {
params.get(PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO, &max_scan_ratio_);
max_scan_num_ =
static_cast<uint32_t>(max_scan_ratio_ * entity_->doc_cnt());
max_scan_num_ = std::max(10000U, max_scan_num_);
}
if (params.has(PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD)) {
params.get(PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
}
return update_visit_filter_param();
case kStreamerContext:
if (params.has(PARAM_HNSW_RABITQ_STREAMER_EF)) {
params.get(PARAM_HNSW_RABITQ_STREAMER_EF, &ef_);
topk_heap_.limit(std::max(topk_, ef_));
}
params.get(PARAM_HNSW_RABITQ_STREAMER_EF, &ef_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
if (max_scan_ratio_ <= 0.0f || max_scan_ratio_ > 1.0f) {
LOG_ERROR("[%s] must be in range (0.0f,1.0f]",
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO.c_str());
return IndexError_InvalidArgument;
}
if (max_scan_limit_ < min_scan_limit_) {
LOG_ERROR("[%s] must be >= [%s]",
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT.c_str(),
PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT.c_str());
return IndexError_InvalidArgument;
}
if (params.has(PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD)) {
params.get(PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
}
return update_visit_filter_param();
default:
LOG_ERROR("update context failed, type=%zu", static_cast<size_t>(type_));
return IndexError_Runtime;
}
}
int HnswRabitqContext::update_context(ContextType type, const IndexMeta &meta,
const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity,
uint32_t magic_num) {
uint32_t doc_cnt;
if (ailego_unlikely(type != type_)) {
LOG_ERROR(
"HnswRabitqContext doesn't support shared by different type, "
"src=%u dst=%u",
type_, type);
return IndexError_Unsupported;
}
magic_ = kInvalidMgic;
// TODO: support change filter mode?
switch (type) {
case kBuilderContext:
LOG_ERROR("BuildContext doesn't support update");
return IndexError_NotImplemented;
case kSearcherContext:
if (!visit_filter_.reset(entity->doc_cnt(), max_scan_num_)) {
LOG_ERROR("Reset filter failed, mode %d", visit_filter_.get_mode());
return IndexError_Runtime;
}
candidates_.limit(max_scan_num_);
topk_heap_.limit(std::max(topk_, ef_));
break;
case kStreamerContext:
doc_cnt = entity->doc_cnt();
max_scan_num_ = compute_max_scan_num(doc_cnt);
reserve_max_doc_cnt_ = doc_cnt + compute_reserve_cnt(doc_cnt);
if (!visit_filter_.reset(reserve_max_doc_cnt_, max_scan_num_)) {
LOG_ERROR("Reset filter failed, mode %d", visit_filter_.get_mode());
return IndexError_Runtime;
}
update_heap_.limit(entity->l0_neighbor_cnt() + 1);
candidates_.limit(max_scan_num_);
topk_heap_.limit(std::max(topk_, ef_));
break;
default:
LOG_ERROR("update context failed");
return IndexError_Runtime;
}
entity_ = entity;
dc().update(entity_.get(), metric, meta.dimension());
magic_ = magic_num;
level_topks_.clear();
return 0;
}
void HnswRabitqContext::fill_random_to_topk_full(void) {
static std::mt19937 mt(
std::chrono::system_clock::now().time_since_epoch().count());
std::uniform_int_distribution<node_id_t> dt(0, entity_->doc_cnt() - 1);
std::function<node_id_t()> gen;
node_id_t seqid;
std::function<bool(node_id_t)> myfilter = [](node_id_t) { return false; };
if (this->filter().is_valid()) {
myfilter = [&](node_id_t id) {
return this->filter()(entity_->get_key(id));
};
}
if (topk_heap_.limit() < entity_->doc_cnt() / 2) {
gen = [&](void) { return dt(mt); };
} else {
// If topk limit is big value, gen sequential id from an random initial
seqid = dt(mt);
gen = [&](void) {
seqid = seqid == (entity_->doc_cnt() - 1) ? 0 : (seqid + 1);
return seqid;
};
}
for (size_t i = 0; !topk_heap_.full() && i < entity_->doc_cnt(); ++i) {
const auto id = gen();
if (!visit_filter_.visited(id) && !myfilter(id)) {
visit_filter_.set_visited(id);
topk_heap_.emplace(id, dc().dist(id));
}
}
return;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,541 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/ailego/logger/logger.h>
#include "utility/visit_filter.h"
#include "zvec/core/framework/index_context.h"
#include "zvec/core/framework/index_provider.h"
#include "hnsw_rabitq_dist_calculator.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
class HnswRabitqContext : public IndexContext {
public:
//! Index Context Pointer
typedef std::unique_ptr<HnswRabitqContext> Pointer;
enum ContextType {
kUnknownContext = 0,
kSearcherContext = 1,
kBuilderContext = 2,
kStreamerContext = 3
};
//! Construct
HnswRabitqContext(size_t dimension, const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity);
//! Construct
HnswRabitqContext(const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity);
//! Destructor
virtual ~HnswRabitqContext();
public:
//! Set topk of search result
virtual void set_topk(uint32_t val) override {
topk_ = val;
topk_heap_.limit(std::max(val, ef_));
}
//! Retrieve search result
virtual const IndexDocumentList &result(void) const override {
return results_[0];
}
//! Retrieve search result
virtual const IndexDocumentList &result(size_t idx) const override {
return results_[idx];
}
//! Retrieve result object for output
virtual IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(void) const override {
return group_results_[0];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(
size_t idx) const override {
return group_results_[idx];
}
virtual uint32_t magic(void) const override {
return magic_;
}
//! Set mode of debug
virtual void set_debug_mode(bool enable) override {
debug_mode_ = enable;
}
//! Retrieve mode of debug
virtual bool debug_mode(void) const override {
return this->debugging();
}
//! Retrieve string of debug
virtual std::string debug_string(void) const override {
char buf[4096];
size_t size = snprintf(
buf, sizeof(buf),
"scan_cnt=%zu,get_vector_cnt=%u,get_neighbors_cnt=%u,dup_node=%u",
get_scan_num(), stats_get_vector_cnt_, stats_get_neighbors_cnt_,
stats_visit_dup_cnt_);
return std::string(buf, size);
}
//! Update the parameters of context
virtual int update(const ailego::Params &params) override;
public:
//! Init context
int init(ContextType type);
//! Update context, the context may be shared by different searcher/streamer
int update_context(ContextType type, const IndexMeta &meta,
const IndexMetric::Pointer &metric,
const HnswRabitqEntity::Pointer &entity,
uint32_t magic_num);
inline const HnswRabitqEntity &get_entity() const {
return *entity_;
}
inline void resize_results(size_t size) {
if (group_by_search()) {
group_results_.resize(size);
} else {
results_.resize(size);
}
}
inline void topk_to_result() {
return topk_to_result(0);
}
//! Construct result from topk heap, result will be normalized
inline void topk_to_result(uint32_t idx) {
if (group_by_search()) {
topk_to_group_result(idx);
} else {
topk_to_single_result(idx);
}
}
inline void topk_to_single_result(uint32_t idx) {
if (force_padding_topk_ && !topk_heap_.full() &&
topk_heap_.size() < entity_->doc_cnt()) {
this->fill_random_to_topk_full();
}
if (ailego_unlikely(topk_heap_.size() == 0)) {
return;
}
ailego_assert_with(idx < results_.size(), "invalid idx");
int size = std::min(topk_, static_cast<uint32_t>(topk_heap_.size()));
topk_heap_.sort();
results_[idx].clear();
for (int i = 0; i < size; ++i) {
auto score = topk_heap_[i].second;
if (score.est_dist > this->threshold()) {
break;
}
node_id_t id = topk_heap_[i].first;
if (fetch_vector_) {
results_[idx].emplace_back(entity_->get_key(id), score.est_dist, id,
entity_->get_vector(id));
} else {
results_[idx].emplace_back(entity_->get_key(id), score.est_dist, id);
}
}
return;
}
//! Construct result from topk heap, result will be normalized
inline void topk_to_group_result(uint32_t idx) {
ailego_assert_with(idx < group_results_.size(), "invalid idx");
group_results_[idx].clear();
std::vector<std::pair<std::string, TopkHeap>> group_topk_list;
std::vector<std::pair<std::string, ResultRecord>> best_score_in_groups;
for (auto itr = group_topk_heaps_.begin(); itr != group_topk_heaps_.end();
itr++) {
const std::string &group_id = (*itr).first;
auto &heap = (*itr).second;
heap.sort();
if (heap.size() > 0) {
ResultRecord best_score = heap[0].second;
best_score_in_groups.push_back(std::make_pair(group_id, best_score));
}
}
std::sort(best_score_in_groups.begin(), best_score_in_groups.end(),
[](const std::pair<std::string, ResultRecord> &a,
const std::pair<std::string, ResultRecord> &b) -> int {
return a.second < b.second;
});
// truncate to group num
for (uint32_t i = 0; i < group_num() && i < best_score_in_groups.size();
++i) {
const std::string &group_id = best_score_in_groups[i].first;
group_topk_list.emplace_back(
std::make_pair(group_id, group_topk_heaps_[group_id]));
}
group_results_[idx].resize(group_topk_list.size());
for (uint32_t i = 0; i < group_topk_list.size(); ++i) {
const std::string &group_id = group_topk_list[i].first;
group_results_[idx][i].set_group_id(group_id);
uint32_t size = std::min(
group_topk_, static_cast<uint32_t>(group_topk_list[i].second.size()));
for (uint32_t j = 0; j < size; ++j) {
auto score = group_topk_list[i].second[j].second;
if (score > this->threshold()) {
break;
}
node_id_t id = group_topk_list[i].second[j].first;
if (fetch_vector_) {
group_results_[idx][i].mutable_docs()->emplace_back(
entity_->get_key(id), score.est_dist, id,
entity_->get_vector(id));
} else {
group_results_[idx][i].mutable_docs()->emplace_back(
entity_->get_key(id), score.est_dist, id);
}
}
}
}
inline void reset_query(const void *query) {
if (auto query_preprocess_func = index_metric_->get_query_preprocess_func();
query_preprocess_func != nullptr) {
size_t dim = dc().dimension();
preprocess_buffer_.resize(dim);
memcpy(preprocess_buffer_.data(), query, dim);
query_preprocess_func(preprocess_buffer_.data(), dim);
query = preprocess_buffer_.data();
}
dc().reset_query(query);
dc().clear_compare_cnt();
query_ = query;
}
inline HnswRabitqAddDistCalculator &dist_calculator() {
return dc();
}
inline TopkHeap &topk_heap() {
return topk_heap_;
}
inline TopkHeap &update_heap() {
return update_heap_;
}
inline VisitFilter &visit_filter() {
return visit_filter_;
}
inline CandidateHeap &candidates() {
return candidates_;
}
inline void set_max_scan_num(uint32_t max_scan_num) {
max_scan_num_ = max_scan_num;
}
inline void set_max_scan_limit(uint32_t max_scan_limit) {
max_scan_limit_ = max_scan_limit;
}
inline void set_min_scan_limit(uint32_t min_scan_limit) {
min_scan_limit_ = min_scan_limit;
}
inline void set_ef(uint32_t v) {
ef_ = v;
}
inline void set_filter_mode(uint32_t v) {
filter_mode_ = v;
}
inline void set_filter_negative_probability(float v) {
negative_probability_ = v;
}
inline void set_max_scan_ratio(float v) {
max_scan_ratio_ = v;
}
virtual void set_magic(uint32_t v) {
magic_ = v;
}
virtual void set_force_padding_topk(bool v) {
force_padding_topk_ = v;
}
void set_bruteforce_threshold(uint32_t v) override {
bruteforce_threshold_ = v;
}
inline uint32_t get_bruteforce_threshold() const {
return bruteforce_threshold_;
}
void set_fetch_vector(bool v) override {
fetch_vector_ = v;
}
bool fetch_vector() const override {
return fetch_vector_;
}
//! Reset context
void reset(void) override {
set_filter(nullptr);
reset_threshold();
set_fetch_vector(false);
set_group_params(0, 0);
reset_group_by();
}
inline std::map<std::string, TopkHeap> &group_topk_heaps() {
return group_topk_heaps_;
}
inline TopkHeap &level_topk(int level) {
if (ailego_unlikely(level_topks_.size() <= static_cast<size_t>(level))) {
int cur_level = level_topks_.size();
level_topks_.resize(level + 1);
for (; cur_level <= level; ++cur_level) {
size_t heap_size = std::max(entity_->neighbor_cnt(cur_level),
entity_->ef_construction());
level_topks_[cur_level].clear();
level_topks_[cur_level].limit(heap_size);
}
}
return level_topks_[level];
}
inline void check_need_adjuct_ctx(void) {
check_need_adjuct_ctx(entity_->doc_cnt());
}
inline size_t compute_reserve_cnt(uint32_t cur_doc) const {
if (cur_doc > kMaxReserveDocCnt) {
return kMaxReserveDocCnt;
} else if (cur_doc < kMinReserveDocCnt) {
return kMinReserveDocCnt;
}
return cur_doc;
}
//! candidates heap and visitfilter need to resize as doc cnt growing up
inline void check_need_adjuct_ctx(uint32_t doc_cnt) {
if (ailego_unlikely(doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_)) {
while (doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_) {
reserve_max_doc_cnt_ =
reserve_max_doc_cnt_ + compute_reserve_cnt(reserve_max_doc_cnt_);
}
uint32_t max_scan_cnt = compute_max_scan_num(reserve_max_doc_cnt_);
max_scan_num_ = max_scan_cnt;
visit_filter_.reset(reserve_max_doc_cnt_, max_scan_cnt);
candidates_.clear();
candidates_.limit(max_scan_num_);
}
}
inline uint32_t compute_max_scan_num(uint32_t max_doc_cnt) const {
uint32_t max_scan = max_doc_cnt * max_scan_ratio_;
if (max_scan < min_scan_limit_) {
max_scan = min_scan_limit_;
} else if (max_scan > max_scan_limit_) {
max_scan = max_scan_limit_;
}
return max_scan;
}
inline size_t get_scan_num() const {
return dc().compare_cnt();
}
inline uint64_t reach_scan_limit() const {
return dc().compare_cnt() >= max_scan_num_;
}
inline bool error() const {
return dc().error();
}
inline void clear() {
add_dc_.clear();
if (ailego_unlikely(this->debugging())) {
stats_get_neighbors_cnt_ = 0u;
stats_get_vector_cnt_ = 0u;
stats_visit_dup_cnt_ = 0u;
}
// do not clear results_ for the next query will need it
for (auto &it : results_) {
it.clear();
}
}
uint32_t *mutable_stats_get_neighbors() {
return &stats_get_neighbors_cnt_;
}
uint32_t *mutable_stats_get_vector() {
return &stats_get_vector_cnt_;
}
uint32_t *mutable_stats_visit_dup_cnt() {
return &stats_visit_dup_cnt_;
}
inline bool debugging(void) const {
return debug_mode_;
}
inline void update_dist_caculator_distance(
const IndexMetric::MatrixDistance &distance,
const IndexMetric::MatrixBatchDistance &batch_distance) {
dc().update_distance(distance, batch_distance);
}
//! Get topk
inline uint32_t topk() const override {
return topk_;
}
//! Get group topk
inline uint32_t group_topk() const {
return group_topk_;
}
//! Get group num
inline uint32_t group_num() const {
return group_num_;
}
//! Get if group by search
inline bool group_by_search() {
return group_num_ > 0;
}
//! Set group params
void set_group_params(uint32_t group_num, uint32_t group_topk) override {
group_num_ = group_num;
group_topk_ = group_topk;
topk_ = group_topk_ * group_num_;
topk_heap_.limit(std::max(topk_, ef_));
group_topk_heaps_.clear();
}
void set_provider(IndexProvider::Pointer provider) {
add_dc_.set_provider(std::move(provider));
}
const void *query() const {
return query_;
}
private:
inline HnswRabitqAddDistCalculator &dc() {
return add_dc_;
}
inline const HnswRabitqAddDistCalculator &dc() const {
return add_dc_;
}
private:
// Filling random nodes if topk not full
void fill_random_to_topk_full(void);
constexpr static uint32_t kTriggerReserveCnt = 4096UL;
constexpr static uint32_t kMinReserveDocCnt = 4096UL;
constexpr static uint32_t kMaxReserveDocCnt = 128 * 1024UL;
constexpr static uint32_t kInvalidMgic = -1U;
private:
HnswRabitqEntity::Pointer entity_;
HnswRabitqAddDistCalculator add_dc_;
IndexMetric::Pointer metric_;
bool debug_mode_{false};
bool force_padding_topk_{false};
uint32_t max_scan_num_{0};
uint32_t max_scan_limit_{0};
uint32_t min_scan_limit_{0};
uint32_t reserve_max_doc_cnt_{kMinReserveDocCnt};
uint32_t topk_{0};
uint32_t group_topk_{0};
uint32_t filter_mode_{VisitFilter::ByteMap};
float negative_probability_{HnswRabitqEntity::kDefaultBFNegativeProbability};
uint32_t ef_{HnswRabitqEntity::kDefaultEf};
float max_scan_ratio_{HnswRabitqEntity::kDefaultScanRatio};
uint32_t magic_{0U};
std::vector<IndexDocumentList> results_{};
std::vector<IndexGroupDocumentList> group_results_{};
TopkHeap topk_heap_{};
TopkHeap update_heap_{};
std::vector<TopkHeap> level_topks_{};
CandidateHeap candidates_{};
VisitFilter visit_filter_{};
uint32_t bruteforce_threshold_{};
bool fetch_vector_{false};
uint32_t group_num_{0};
std::map<std::string, TopkHeap> group_topk_heaps_{};
uint32_t type_{kUnknownContext};
//! debug stats info
uint32_t stats_get_neighbors_cnt_{0u};
uint32_t stats_get_vector_cnt_{0u};
uint32_t stats_visit_dup_cnt_{0u};
std::string preprocess_buffer_;
const void *query_{nullptr};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,39 @@
// Copyright 2025-present the centaurdb project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
#include "core/algorithm/hnsw_rabitq/hnsw_rabitq_dist_calculator.h"
#include "zvec/core/framework/index_error.h"
namespace zvec::core {
int HnswRabitqAddDistCalculator::get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
for (uint32_t i = 0; i < count; ++i) {
const node_id_t id = ids[i];
key_t key = entity_->get_key(id);
if (key == kInvalidKey) {
return IndexError_NoExist;
}
IndexStorage::MemoryBlock block;
int ret = provider_->get_vector(key, block);
if (ret != 0) {
return ret;
}
vec_blocks.push_back(std::move(block));
}
return 0;
}
} // namespace zvec::core
@@ -0,0 +1,240 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "zvec/core/framework/index_meta.h"
#include "zvec/core/framework/index_metric.h"
#include "zvec/core/framework/index_provider.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
//! HnswRabitqAddDistCalculator is only used for index construction
class HnswRabitqAddDistCalculator {
public:
typedef std::shared_ptr<HnswRabitqAddDistCalculator> Pointer;
public:
enum DistType {
DIST_NONE = 0,
DIST_DENSE = 1,
DIST_HYBRID = 2,
DIST_SPARSE = 3
};
public:
//! Constructor
HnswRabitqAddDistCalculator(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric, uint32_t dim)
: entity_(entity),
distance_(metric->distance()),
batch_distance_(metric->batch_distance()),
query_(nullptr),
dim_(dim),
compare_cnt_(0) {}
//! Constructor
HnswRabitqAddDistCalculator(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric, uint32_t dim,
const void *query)
: entity_(entity),
distance_(metric->distance()),
batch_distance_(metric->batch_distance()),
query_(query),
dim_(dim),
compare_cnt_(0) {}
//! Constructor
HnswRabitqAddDistCalculator(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric)
: entity_(entity),
distance_(metric->distance()),
batch_distance_(metric->batch_distance()),
query_(nullptr),
dim_(0),
compare_cnt_(0) {}
void update(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric) {
entity_ = entity;
distance_ = metric->distance();
batch_distance_ = metric->batch_distance();
}
void update(const HnswRabitqEntity *entity,
const IndexMetric::Pointer &metric, uint32_t dim) {
entity_ = entity;
distance_ = metric->distance();
batch_distance_ = metric->batch_distance();
dim_ = dim;
}
inline void update_distance(
const IndexMetric::MatrixDistance &distance,
const IndexMetric::MatrixBatchDistance &batch_distance) {
distance_ = distance;
batch_distance_ = batch_distance;
}
//! Reset query vector data
inline void reset_query(const void *query) {
error_ = false;
query_ = query;
}
//! Returns distance
inline dist_t dist(const void *vec_lhs, const void *vec_rhs) {
if (ailego_unlikely(vec_lhs == nullptr || vec_rhs == nullptr)) {
LOG_ERROR("Nullptr of dense vector");
error_ = true;
return 0.0f;
}
float score{0.0f};
distance_(vec_lhs, vec_rhs, dim_, &score);
return score;
}
//! Returns distance between query and vec.
inline dist_t dist(const void *vec) {
compare_cnt_++;
return dist(vec, query_);
}
//! Return distance between query and node id.
inline dist_t dist(node_id_t id) {
compare_cnt_++;
const void *feat = get_vector(id);
if (ailego_unlikely(feat == nullptr)) {
LOG_ERROR("Get nullptr vector, id=%u", id);
error_ = true;
return 0.0f;
}
return dist(feat, query_);
}
//! Return dist node lhs between node rhs
inline dist_t dist(node_id_t lhs, node_id_t rhs) {
compare_cnt_++;
const void *feat = get_vector(lhs);
const void *query = get_vector(rhs);
if (ailego_unlikely(feat == nullptr || query == nullptr)) {
LOG_ERROR("Get nullptr vector");
error_ = true;
return 0.0f;
}
return dist(feat, query);
}
dist_t operator()(const void *vec) {
return dist(vec);
}
dist_t operator()(id_t i) {
return dist(i);
}
dist_t operator()(id_t lhs, id_t rhs) {
return dist(lhs, rhs);
}
void batch_dist(const void **vecs, size_t num, dist_t *distances) {
compare_cnt_++;
batch_distance_(vecs, query_, num, dim_, distances);
}
inline dist_t batch_dist(node_id_t id) {
compare_cnt_++;
const void *feat = get_vector(id);
if (ailego_unlikely(feat == nullptr)) {
LOG_ERROR("Get nullptr vector, id=%u", id);
error_ = true;
return 0.0f;
}
dist_t score = 0;
batch_distance_(&feat, query_, 1, dim_, &score);
return score;
}
inline void clear() {
compare_cnt_ = 0;
error_ = false;
}
inline void clear_compare_cnt() {
compare_cnt_ = 0;
}
inline bool error() const {
return error_;
}
//! Get distances compute times
inline uint32_t compare_cnt() const {
return compare_cnt_;
}
inline uint32_t dimension() const {
return dim_;
}
void set_provider(IndexProvider::Pointer provider) {
provider_ = std::move(provider);
}
int get_vector(const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const;
const void *get_vector(node_id_t id) const {
key_t key = entity_->get_key(id);
if (key == kInvalidKey) {
return nullptr;
}
return provider_->get_vector(key);
}
private:
HnswRabitqAddDistCalculator(const HnswRabitqAddDistCalculator &) = delete;
HnswRabitqAddDistCalculator &operator=(const HnswRabitqAddDistCalculator &) =
delete;
private:
const HnswRabitqEntity *entity_;
IndexMetric::MatrixDistance distance_;
IndexMetric::MatrixBatchDistance batch_distance_;
const void *query_;
uint32_t dim_;
uint32_t compare_cnt_; // record distance compute times
uint32_t compare_cnt_batch_; // record batch distance compute time
bool error_{false};
// get raw vector
IndexProvider::Pointer provider_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,382 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_entity.h"
#include <rabitqlib/index/query.hpp>
#include "utility/sparse_utility.h"
#include "zvec/core/framework/index_stats.h"
namespace zvec {
namespace core {
const std::string HnswRabitqEntity::kGraphHeaderSegmentId = "graph.header";
const std::string HnswRabitqEntity::kGraphFeaturesSegmentId = "graph.features";
const std::string HnswRabitqEntity::kGraphKeysSegmentId = "graph.keys";
const std::string HnswRabitqEntity::kGraphNeighborsSegmentId =
"graph.neighbors";
const std::string HnswRabitqEntity::kGraphOffsetsSegmentId = "graph.offsets";
const std::string HnswRabitqEntity::kGraphMappingSegmentId = "graph.mapping";
const std::string HnswRabitqEntity::kHnswHeaderSegmentId = "hnsw.header";
const std::string HnswRabitqEntity::kHnswNeighborsSegmentId = "hnsw.neighbors";
const std::string HnswRabitqEntity::kHnswOffsetsSegmentId = "hnsw.offsets";
void HnswRabitqEntity::update_rabitq_params_and_vector_size(
uint32_t dimension) {
uint32_t padded_dim = ((dimension + 63) / 64) * 64;
header_.graph.padded_dim = padded_dim;
// BinDataMap layout: bin_code (padded_dim/8) + f_add + f_rescale + f_error
header_.graph.size_bin_data =
rabitqlib::BinDataMap<float>::data_bytes(padded_dim);
// ExDataMap layout: ex_code (padded_dim*ex_bits/8) + f_add_ex + f_rescale_ex
header_.graph.size_ex_data = rabitqlib::ExDataMap<float>::data_bytes(
padded_dim, header_.graph.ex_bits);
// quantized vector format: cluster_id + bin_data + ex_data
header_.graph.vector_size =
sizeof(uint32_t) + size_bin_data() + size_ex_data();
}
int HnswRabitqEntity::CalcAndAddPadding(const IndexDumper::Pointer &dumper,
size_t data_size,
size_t *padding_size) {
*padding_size = AlignSize(data_size) - data_size;
if (*padding_size == 0) {
return 0;
}
std::string padding(*padding_size, '\0');
if (dumper->write(padding.data(), *padding_size) != *padding_size) {
LOG_ERROR("Append padding failed, size %zu", *padding_size);
return IndexError_WriteData;
}
return 0;
}
int64_t HnswRabitqEntity::dump_segment(const IndexDumper::Pointer &dumper,
const std::string &segment_id,
const void *data, size_t size) const {
size_t len = dumper->write(data, size);
if (len != size) {
LOG_ERROR("Dump segment %s data failed, expect: %zu, actual: %zu",
segment_id.c_str(), size, len);
return IndexError_WriteData;
}
size_t padding_size = AlignSize(size) - size;
if (padding_size > 0) {
std::string padding(padding_size, '\0');
if (dumper->write(padding.data(), padding_size) != padding_size) {
LOG_ERROR("Append padding failed, size %zu", padding_size);
return IndexError_WriteData;
}
}
uint32_t crc = ailego::Crc32c::Hash(data, size);
int ret = dumper->append(segment_id, size, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
return ret;
}
return len + padding_size;
}
int64_t HnswRabitqEntity::dump_header(const IndexDumper::Pointer &dumper,
const HNSWHeader &hd) const {
//! dump basic graph header. header is aligned and does not need padding
int64_t graph_hd_size =
dump_segment(dumper, kGraphHeaderSegmentId, &hd.graph, hd.graph.size);
if (graph_hd_size < 0) {
return graph_hd_size;
}
//! dump basic graph header. header is aligned and does not need padding
int64_t hnsw_hd_size =
dump_segment(dumper, kHnswHeaderSegmentId, &hd.hnsw, hd.hnsw.size);
if (hnsw_hd_size < 0) {
return hnsw_hd_size;
}
return graph_hd_size + hnsw_hd_size;
}
void HnswRabitqEntity::reshuffle_vectors(
const std::function<level_t(node_id_t)> & /*get_level*/,
std::vector<node_id_t> * /*n2o_mapping*/,
std::vector<node_id_t> * /*o2n_mapping*/, key_t * /*keys*/) const {
// TODO
return;
}
int64_t HnswRabitqEntity::dump_mapping_segment(
const IndexDumper::Pointer &dumper, const key_t *keys) const {
std::vector<node_id_t> mapping(doc_cnt());
std::iota(mapping.begin(), mapping.end(), 0U);
std::sort(mapping.begin(), mapping.end(),
[&](node_id_t i, node_id_t j) { return keys[i] < keys[j]; });
size_t size = mapping.size() * sizeof(node_id_t);
return dump_segment(dumper, kGraphMappingSegmentId, mapping.data(), size);
}
int64_t HnswRabitqEntity::dump_segments(
const IndexDumper::Pointer &dumper, key_t *keys,
const std::function<level_t(node_id_t)> &get_level) const {
HNSWHeader dump_hd(header());
dump_hd.graph.node_size = AlignSize(vector_size());
std::vector<node_id_t> n2o_mapping; // map new id to origin id
std::vector<node_id_t> o2n_mapping; // map origin id to new id
reshuffle_vectors(get_level, &n2o_mapping, &o2n_mapping, keys);
if (!o2n_mapping.empty()) {
dump_hd.hnsw.entry_point = o2n_mapping[entry_point()];
}
//! Dump header
int64_t hd_size = dump_header(dumper, dump_hd);
if (hd_size < 0) {
return hd_size;
}
//! Dump vectors
int64_t vecs_size = dump_vectors(dumper, n2o_mapping);
if (vecs_size < 0) {
return vecs_size;
}
//! Dump neighbors
auto neighbors_size =
dump_neighbors(dumper, get_level, n2o_mapping, o2n_mapping);
if (neighbors_size < 0) {
return neighbors_size;
}
//! free memory
n2o_mapping = std::vector<node_id_t>();
o2n_mapping = std::vector<node_id_t>();
//! Dump keys
size_t key_segment_size = doc_cnt() * sizeof(key_t);
int64_t keys_size =
dump_segment(dumper, kGraphKeysSegmentId, keys, key_segment_size);
if (keys_size < 0) {
return keys_size;
}
//! Dump mapping
int64_t mapping_size = dump_mapping_segment(dumper, keys);
if (mapping_size < 0) {
return mapping_size;
}
return hd_size + keys_size + vecs_size + neighbors_size + mapping_size;
}
int64_t HnswRabitqEntity::dump_vectors(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping) const {
size_t vector_dump_size = vector_size();
size_t padding_size = AlignSize(vector_dump_size) - vector_dump_size;
char padding[padding_size];
memset(padding, 0, sizeof(padding));
const void *data = nullptr;
uint32_t crc = 0U;
size_t vecs_size = 0UL;
//! dump vectors
for (node_id_t id = 0; id < doc_cnt(); ++id) {
data = get_vector(reorder_mapping.empty() ? id : reorder_mapping[id]);
if (ailego_unlikely(!data)) {
return IndexError_ReadData;
}
size_t len = dumper->write(data, vector_size());
if (len != vector_size()) {
LOG_ERROR("Dump vectors failed, write=%zu expect=%zu", len,
vector_size());
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(data, vector_size(), crc);
vecs_size += vector_size();
if (padding_size == 0) {
continue;
}
len = dumper->write(padding, padding_size);
if (len != padding_size) {
LOG_ERROR("Dump vectors failed, write=%zu expect=%zu", len, padding_size);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(padding, padding_size, crc);
vecs_size += padding_size;
}
int ret = dumper->append(kGraphFeaturesSegmentId, vecs_size, 0UL, crc);
if (ret != 0) {
LOG_ERROR("Dump vectors segment meta failed, ret %d", ret);
return ret;
}
return vecs_size;
}
int64_t HnswRabitqEntity::dump_graph_neighbors(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const {
std::vector<GraphNeighborMeta> graph_meta;
graph_meta.reserve(doc_cnt());
size_t offset = 0;
uint32_t crc = 0;
node_id_t mapping[l0_neighbor_cnt()];
uint32_t min_neighbor_count = 10000;
uint32_t max_neighbor_count = 0;
size_t sum_neighbor_count = 0;
for (node_id_t id = 0; id < doc_cnt(); ++id) {
const Neighbors neighbors =
get_neighbors(0, reorder_mapping.empty() ? id : reorder_mapping[id]);
ailego_assert_with(!!neighbors.data, "invalid neighbors");
ailego_assert_with(neighbors.size() <= l0_neighbor_cnt(),
"invalid neighbors");
uint32_t neighbor_count = neighbors.size();
if (neighbor_count < min_neighbor_count) {
min_neighbor_count = neighbor_count;
}
if (neighbor_count > max_neighbor_count) {
max_neighbor_count = neighbor_count;
}
sum_neighbor_count += neighbor_count;
graph_meta.emplace_back(offset, neighbor_count);
size_t size = neighbors.size() * sizeof(node_id_t);
const node_id_t *data = &neighbors[0];
if (!neighbor_mapping.empty()) {
for (node_id_t i = 0; i < neighbors.size(); ++i) {
mapping[i] = neighbor_mapping[neighbors[i]];
}
data = mapping;
}
if (dumper->write(data, size) != size) {
LOG_ERROR("Dump graph neighbor id=%zu failed, size %zu",
static_cast<size_t>(id), size);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(data, size, crc);
offset += size;
}
uint32_t average_neighbor_count = 0;
if (doc_cnt() > 0) {
average_neighbor_count = sum_neighbor_count / doc_cnt();
}
LOG_INFO(
"Dump hnsw graph: min_neighbor_count[%u] max_neighbor_count[%u] "
"average_neighbor_count[%u]",
min_neighbor_count, max_neighbor_count, average_neighbor_count);
size_t padding_size = 0;
int ret = CalcAndAddPadding(dumper, offset, &padding_size);
if (ret != 0) {
return ret;
}
ret = dumper->append(kGraphNeighborsSegmentId, offset, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s failed, ret %d",
kGraphNeighborsSegmentId.c_str(), ret);
return ret;
}
//! dump level 0 neighbors meta
auto len = dump_segment(dumper, kGraphOffsetsSegmentId, graph_meta.data(),
graph_meta.size() * sizeof(GraphNeighborMeta));
if (len < 0) {
return len;
}
return len + offset + padding_size;
}
int64_t HnswRabitqEntity::dump_upper_neighbors(
const IndexDumper::Pointer &dumper,
const std::function<level_t(node_id_t)> &get_level,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const {
std::vector<HnswNeighborMeta> hnsw_meta;
hnsw_meta.reserve(doc_cnt());
size_t offset = 0;
uint32_t crc = 0;
node_id_t buffer[upper_neighbor_cnt() + 1];
for (node_id_t id = 0; id < doc_cnt(); ++id) {
node_id_t new_id = reorder_mapping.empty() ? id : reorder_mapping[id];
auto level = get_level(new_id);
if (level == 0) {
hnsw_meta.emplace_back(0U, 0U);
continue;
}
hnsw_meta.emplace_back(offset, level);
ailego_assert_with((size_t)level < kMaxGraphLayers, "invalid level");
for (level_t cur_level = 1; cur_level <= level; ++cur_level) {
const Neighbors neighbors = get_neighbors(cur_level, new_id);
ailego_assert_with(!!neighbors.data, "invalid neighbors");
ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level),
"invalid neighbors");
memset(buffer, 0, sizeof(buffer));
buffer[0] = neighbors.size();
if (neighbor_mapping.empty()) {
memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t));
} else {
for (node_id_t i = 0; i < neighbors.size(); ++i) {
buffer[i + 1] = neighbor_mapping[neighbors[i]];
}
}
if (dumper->write(buffer, sizeof(buffer)) != sizeof(buffer)) {
LOG_ERROR("Dump graph neighbor id=%zu failed, size %zu",
static_cast<size_t>(id), sizeof(buffer));
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(buffer, sizeof(buffer), crc);
offset += sizeof(buffer);
}
}
size_t padding_size = 0;
int ret = CalcAndAddPadding(dumper, offset, &padding_size);
if (ret != 0) {
return ret;
}
ret = dumper->append(kHnswNeighborsSegmentId, offset, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s failed, ret %d", kHnswNeighborsSegmentId.c_str(),
ret);
return ret;
}
//! dump level 0 neighbors meta
auto len = dump_segment(dumper, kHnswOffsetsSegmentId, hnsw_meta.data(),
hnsw_meta.size() * sizeof(HnswNeighborMeta));
if (len < 0) {
return len;
}
return len + offset + padding_size;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,614 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <execinfo.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string>
#include <ailego/utility/memory_helper.h>
#include <zvec/ailego/container/heap.h>
#include <zvec/ailego/logger/logger.h>
#include "zvec/core/framework/index_dumper.h"
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_storage.h"
namespace zvec {
namespace core {
using node_id_t = uint32_t;
using key_t = uint64_t;
using level_t = int32_t;
using dist_t = float;
struct EstimateRecord {
float ip_x0_qr;
float est_dist;
float low_dist;
bool operator<(const EstimateRecord &other) const {
return this->est_dist < other.est_dist;
}
};
struct ResultRecord {
float est_dist;
float low_dist;
ResultRecord() : est_dist(0.0f), low_dist(0.0f) {}
ResultRecord(float dist) : est_dist(dist), low_dist(dist) {}
explicit ResultRecord(const EstimateRecord &other)
: est_dist(other.est_dist), low_dist(other.low_dist) {}
ResultRecord(float est_dist, float low_dist)
: est_dist(est_dist), low_dist(low_dist) {}
bool operator<(const ResultRecord &other) const {
return this->est_dist < other.est_dist;
}
bool operator<=(const ResultRecord &other) const {
return this->est_dist <= other.est_dist;
}
bool operator>(const ResultRecord &other) const {
return this->est_dist > other.est_dist;
}
};
using TopkHeap = ailego::KeyValueHeap<node_id_t, ResultRecord>;
using CandidateHeap =
ailego::KeyValueHeap<node_id_t, ResultRecord, std::greater<ResultRecord>>;
constexpr node_id_t kInvalidNodeId = static_cast<node_id_t>(-1);
constexpr key_t kInvalidKey = static_cast<key_t>(-1);
class DistCalculator;
struct GraphHeader {
uint32_t size;
uint32_t version;
uint32_t graph_type;
uint32_t doc_count;
uint32_t vector_size;
uint32_t node_size;
uint32_t l0_neighbor_count;
uint32_t prune_type;
uint32_t prune_neighbor_count;
uint32_t ef_construction;
uint32_t options;
uint32_t min_neighbor_count;
uint32_t padded_dim;
uint32_t size_bin_data;
uint32_t size_ex_data;
uint8_t ex_bits;
uint8_t reserved_[4067];
};
static_assert(sizeof(GraphHeader) % 32 == 0,
"GraphHeader must be aligned with 32 bytes");
//! Hnsw upper neighbor header
struct HnswHeader {
uint32_t size; // header size
uint32_t revision; // current total docs of the graph
uint32_t upper_neighbor_count;
uint32_t ef_construction;
uint32_t scaling_factor;
uint32_t max_level;
uint32_t entry_point;
uint32_t options;
uint8_t reserved_[30];
};
static_assert(sizeof(HnswHeader) % 32 == 0,
"GraphHeader must be aligned with 32 bytes");
//! Hnsw common header and upper neighbor header
struct HNSWHeader {
HNSWHeader() {
clear();
}
HNSWHeader(const HNSWHeader &header) {
memcpy(this, &header, sizeof(header));
}
HNSWHeader &operator=(const HNSWHeader &header) {
memcpy(this, &header, sizeof(header));
return *this;
}
//! Reset state to zero, and the params is untouched
void inline reset() {
graph.doc_count = 0U;
hnsw.entry_point = kInvalidNodeId;
hnsw.max_level = 0;
}
//! Clear all fields to init value
void inline clear() {
memset(this, 0, sizeof(HNSWHeader));
hnsw.entry_point = kInvalidNodeId;
graph.size = sizeof(GraphHeader);
hnsw.size = sizeof(HnswHeader);
}
size_t l0_neighbor_cnt() const {
return graph.l0_neighbor_count;
}
size_t upper_neighbor_cnt() const {
return hnsw.upper_neighbor_count;
}
size_t vector_size() const {
return graph.vector_size;
}
uint8_t ex_bits() const {
return graph.ex_bits;
}
uint32_t padded_dim() const {
return graph.padded_dim;
}
size_t ef_construction() const {
return graph.ef_construction;
}
size_t scaling_factor() const {
return hnsw.scaling_factor;
}
size_t neighbor_prune_cnt() const {
return graph.prune_neighbor_count;
}
node_id_t entry_point() const {
return hnsw.entry_point;
}
node_id_t doc_cnt() const {
return graph.doc_count;
}
GraphHeader graph;
HnswHeader hnsw;
};
struct NeighborsHeader {
uint32_t neighbor_cnt;
node_id_t neighbors[0];
};
struct Neighbors {
Neighbors() : cnt{0}, data{nullptr} {}
Neighbors(uint32_t cnt_in, const node_id_t *data_in)
: cnt{cnt_in}, data{data_in} {}
Neighbors(IndexStorage::MemoryBlock &&mem_block)
: neighbor_block{std::move(mem_block)} {
auto hd = reinterpret_cast<const NeighborsHeader *>(neighbor_block.data());
cnt = hd->neighbor_cnt;
data = hd->neighbors;
}
size_t size(void) const {
return cnt;
}
const node_id_t &operator[](size_t idx) const {
return data[idx];
}
uint32_t cnt;
const node_id_t *data;
IndexStorage::MemoryBlock neighbor_block;
};
//! level 0 neighbors offset
struct GraphNeighborMeta {
GraphNeighborMeta(size_t o, size_t cnt) : offset(o), neighbor_cnt(cnt) {}
uint64_t offset : 48;
uint64_t neighbor_cnt : 16;
};
//! hnsw upper neighbors meta
struct HnswNeighborMeta {
HnswNeighborMeta(size_t o, size_t l) : offset(o), level(l) {}
uint64_t offset : 48; // offset = idx * upper neighors size
uint64_t level : 16;
};
class HnswRabitqEntity {
public:
//! Constructor
HnswRabitqEntity() {}
//! Constructor
HnswRabitqEntity(const HNSWHeader &hd) {
header_ = hd;
}
//! Destructor
virtual ~HnswRabitqEntity() {}
//! HnswRabitqEntity Pointerd;
typedef std::shared_ptr<HnswRabitqEntity> Pointer;
//! Get max neighbor size of graph level
inline size_t neighbor_cnt(level_t level) const {
return level == 0 ? header_.graph.l0_neighbor_count
: header_.hnsw.upper_neighbor_count;
}
//! get max neighbor size of graph level 0
inline size_t l0_neighbor_cnt() const {
return header_.graph.l0_neighbor_count;
}
//! get min neighbor size of graph
inline size_t min_neighbor_cnt() const {
return header_.graph.min_neighbor_count;
}
//! get upper neighbor size of graph level other than 0
inline size_t upper_neighbor_cnt() const {
return header_.hnsw.upper_neighbor_count;
}
//! Get current total doc of the hnsw graph
inline node_id_t *mutable_doc_cnt() {
return &header_.graph.doc_count;
}
inline node_id_t doc_cnt() const {
return header_.graph.doc_count;
}
//! Get hnsw graph scaling params
inline size_t scaling_factor() const {
return header_.hnsw.scaling_factor;
}
//! Get prune_size
inline size_t prune_cnt() const {
return header_.graph.prune_neighbor_count;
}
//! Current entity of top level graph
inline node_id_t entry_point() const {
return header_.hnsw.entry_point;
}
//! Current max graph level
inline level_t cur_max_level() const {
return header_.hnsw.max_level;
}
//! Retrieve index vector size
size_t vector_size() const {
return header_.graph.vector_size;
}
//! Retrieve node size
size_t node_size() const {
return header_.graph.node_size;
}
//! Retrieve ef constuction
size_t ef_construction() const {
return header_.graph.ef_construction;
}
uint8_t ex_bits() const {
return header_.graph.ex_bits;
}
uint32_t padded_dim() const {
return header_.graph.padded_dim;
}
uint32_t size_bin_data() const {
return header_.graph.size_bin_data;
}
uint32_t size_ex_data() const {
return header_.graph.size_ex_data;
}
void update_rabitq_params_and_vector_size(uint32_t dimension);
void set_ex_bits(uint8_t ex_bits) {
header_.graph.ex_bits = ex_bits;
}
void set_prune_cnt(size_t v) {
header_.graph.prune_neighbor_count = v;
}
void set_scaling_factor(size_t val) {
header_.hnsw.scaling_factor = val;
}
void set_l0_neighbor_cnt(size_t cnt) {
header_.graph.l0_neighbor_count = cnt;
}
void set_min_neighbor_cnt(size_t cnt) {
header_.graph.min_neighbor_count = cnt;
}
void set_upper_neighbor_cnt(size_t cnt) {
header_.hnsw.upper_neighbor_count = cnt;
}
void set_ef_construction(size_t ef) {
header_.graph.ef_construction = ef;
}
protected:
inline const HNSWHeader &header() const {
return header_;
}
inline HNSWHeader *mutable_header() {
return &header_;
}
inline size_t header_size() const {
return sizeof(header_);
}
void set_node_size(size_t size) {
header_.graph.node_size = size;
}
//! Dump all segment by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_segments(
const IndexDumper::Pointer &dumper, key_t *keys,
const std::function<level_t(node_id_t)> &get_level) const;
private:
//! dump mapping segment, for get_vector_by_key in provider
int64_t dump_mapping_segment(const IndexDumper::Pointer &dumper,
const key_t *keys) const;
//! dump hnsw head by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_header(const IndexDumper::Pointer &dumper,
const HNSWHeader &hd) const;
//! dump vectors by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_vectors(const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping) const;
//! dump hnsw neighbors by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_neighbors(const IndexDumper::Pointer &dumper,
const std::function<level_t(node_id_t)> &get_level,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const {
auto len1 = dump_graph_neighbors(dumper, reorder_mapping, neighbor_mapping);
if (len1 < 0) {
return len1;
}
auto len2 = dump_upper_neighbors(dumper, get_level, reorder_mapping,
neighbor_mapping);
if (len2 < 0) {
return len2;
}
return len1 + len2;
}
//! dump segment by dumper
//! Return dump size if success, errno(<0) in failure
int64_t dump_segment(const IndexDumper::Pointer &dumper,
const std::string &segment_id, const void *data,
size_t size) const;
//! Dump level 0 neighbors
//! Return dump size if success, errno(<0) in failure
int64_t dump_graph_neighbors(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const;
//! Dump upper level neighbors
//! Return dump size if success, errno(<0) in failure
int64_t dump_upper_neighbors(
const IndexDumper::Pointer &dumper,
const std::function<level_t(node_id_t)> &get_level,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const;
public:
//! Cleanup the entity
virtual int cleanup(void) {
header_.clear();
return 0;
}
//! Make a copy of searcher entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswRabitqEntity::Pointer clone() const {
LOG_ERROR("Update neighbors not implemented");
return HnswRabitqEntity::Pointer();
}
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const = 0;
//! Get vector feature data by key
virtual const void *get_vector(node_id_t id) const = 0;
//! Get vectors feature data by keys
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const = 0;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const = 0;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const = 0;
//! Retrieve a vector using a primary key
virtual const void *get_vector_by_key(uint64_t /*key*/) const {
LOG_ERROR("get vector not implemented");
return nullptr;
}
virtual int get_vector_by_key(const key_t /*key*/,
IndexStorage::MemoryBlock & /*block*/) const {
return IndexError_NotImplemented;
}
//! Get the node id's neighbors on graph level
//! Note: the neighbors cannot be modified, using the following
//! method to get WritableNeighbors if want to
virtual const Neighbors get_neighbors(level_t level, node_id_t id) const = 0;
//! Add vector and key to hnsw entity, and local id will be saved in id
virtual int add_vector(level_t /*level*/, key_t /*key*/, const void * /*vec*/,
node_id_t * /*id*/) {
return IndexError_NotImplemented;
}
//! Add vector and id to hnsw entity
virtual int add_vector_with_id(level_t /*level*/, node_id_t /*id*/,
const void * /*vec*/) {
return IndexError_NotImplemented;
}
virtual int update_neighbors(
level_t /*level*/, node_id_t /*id*/,
const std::vector<std::pair<node_id_t, ResultRecord>> & /*neighbors*/) {
LOG_ERROR("Update neighbors dense not implemented");
return 0;
}
//! Append neighbor_id to node id neighbors on level, size is the current
//! neighbors size. Notice: the caller must be ensure the neighbors not full
virtual void add_neighbor(level_t /*level*/, node_id_t /*id*/,
uint32_t /*size*/, node_id_t /*neighbor_id*/) {
LOG_ERROR("Add neighbor not implemented");
}
//! Update entry point and max level
virtual void update_ep_and_level(node_id_t ep, level_t level) {
header_.hnsw.entry_point = ep;
header_.hnsw.max_level = level;
}
virtual int load(const IndexStorage::Pointer & /*container*/,
bool /*check_crc*/) {
LOG_ERROR("Load not implemented");
return IndexError_NotImplemented;
}
virtual int dump(const IndexDumper::Pointer & /*dumper*/) {
LOG_ERROR("Dump not implemented");
return IndexError_NotImplemented;
}
static int CalcAndAddPadding(const IndexDumper::Pointer &dumper,
size_t data_size, size_t *padding_size);
uint32_t get_cluster_id(const void *vec) const {
return *reinterpret_cast<const uint32_t *>(
reinterpret_cast<const char *>(vec) + cluster_id_offset());
}
const char *get_bin_data(const void *vec) const {
return reinterpret_cast<const char *>(vec) + bin_data_offset();
}
const char *get_ex_data(const void *vec) const {
return reinterpret_cast<const char *>(vec) + ex_data_offset();
}
uint32_t cluster_id_offset() const {
return 0;
}
uint32_t bin_data_offset() const {
return cluster_id_offset() + sizeof(uint32_t);
}
uint32_t ex_data_offset() const {
return bin_data_offset() + size_bin_data();
}
protected:
static inline size_t AlignSize(size_t size) {
return (size + 0x1F) & (~0x1F);
}
static inline size_t AlignPageSize(size_t size) {
size_t page_mask = ailego::MemoryHelper::PageSize() - 1;
return (size + page_mask) & (~page_mask);
}
static inline size_t AlignHugePageSize(size_t size) {
size_t page_mask = ailego::MemoryHelper::HugePageSize() - 1;
return (size + page_mask) & (~page_mask);
}
//! rearrange vectors to improve cache locality
void reshuffle_vectors(const std::function<level_t(node_id_t)> &get_level,
std::vector<node_id_t> *n2o_mapping,
std::vector<node_id_t> *o2n_mapping,
key_t *keys) const;
public:
const static std::string kGraphHeaderSegmentId;
const static std::string kGraphFeaturesSegmentId;
const static std::string kGraphKeysSegmentId;
const static std::string kGraphNeighborsSegmentId;
const static std::string kGraphOffsetsSegmentId;
const static std::string kGraphMappingSegmentId;
const static std::string kHnswHeaderSegmentId;
const static std::string kHnswNeighborsSegmentId;
const static std::string kHnswOffsetsSegmentId;
constexpr static uint32_t kRevision = 0U;
constexpr static size_t kMaxGraphLayers = 15;
constexpr static uint32_t kDefaultEfConstruction = 500;
constexpr static uint32_t kDefaultEf = 500;
constexpr static uint32_t kDefaultUpperMaxNeighborCnt = 50; // M of HNSW
constexpr static uint32_t kDefaultL0MaxNeighborCnt = 100;
constexpr static uint32_t kMaxNeighborCnt = 65535;
constexpr static float kDefaultScanRatio = 0.1f;
constexpr static uint32_t kDefaultMinScanLimit = 10000;
constexpr static uint32_t kDefaultMaxScanLimit =
std::numeric_limits<uint32_t>::max();
constexpr static float kDefaultBFNegativeProbability = 0.001f;
constexpr static uint32_t kDefaultScalingFactor = 50U;
constexpr static uint32_t kDefaultBruteForceThreshold = 1000U;
constexpr static uint32_t kDefaultDocsHardLimit = 1 << 30U; // 1 billion
constexpr static float kDefaultDocsSoftLimitRatio = 0.9f;
constexpr static size_t kMaxChunkSize = 0xFFFFFFFF;
constexpr static size_t kDefaultChunkSize = 2UL * 1024UL * 1024UL;
constexpr static size_t kDefaultMaxChunkCnt = 50000UL;
constexpr static float kDefaultNeighborPruneMultiplier =
1.0f; // prune_cnt = upper_max_neighbor_cnt * multiplier
constexpr static float kDefaultL0MaxNeighborCntMultiplier =
2.0f; // l0_max_neighbor_cnt = upper_max_neighbor_cnt * multiplier
protected:
HNSWHeader header_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,231 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "hnsw_rabitq_chunk.h"
namespace zvec {
namespace core {
//! Persistent hashmap implement through open addressing algorithm
template <class Key, class Val, Val EmptyVal = 0U,
typename =
typename std::enable_if<std::is_integral<Key>::value>::type>
class HnswIndexHashMap {
using key_type = Key;
using val_type = Val;
struct Iterator {
key_type first;
val_type second;
};
typedef Iterator *iterator;
typedef Iterator Item;
typedef const Iterator *const_iterator;
class Slot {
public:
Slot(Chunk::Pointer &&chunk, const void *data)
: chunk_(std::move(chunk)),
items_(reinterpret_cast<const Item *>(data)) {}
//! Return a empty loc or the key item loc
Slot(Chunk::Pointer &&chunk, IndexStorage::MemoryBlock &&mem_block)
: chunk_(std::move(chunk)), items_block_(std::move(mem_block)) {
items_ = reinterpret_cast<const Item *>(items_block_.data());
}
const_iterator find(key_type key, uint32_t max_items, uint32_t mask) const {
auto it = &items_[key & mask];
for (auto i = 0U; i < max_items; ++i) {
if (it->first == key || it->second == EmptyVal) {
// LOG_DEBUG("i=%u", i);
return it;
}
++it;
if (it == &items_[max_items]) {
it = &items_[0];
}
}
return nullptr;
}
bool update(const_iterator it) {
uint32_t offset = reinterpret_cast<const uint8_t *>(it) -
reinterpret_cast<const uint8_t *>(&items_[0]);
if (ailego_unlikely(chunk_->write(offset, it, sizeof(Item)) !=
sizeof(Item))) {
LOG_ERROR("Chunk write failed");
return false;
}
return true;
}
private:
Chunk::Pointer chunk_{};
const Item *items_{nullptr}; // point to chunk data
IndexStorage::MemoryBlock items_block_{};
};
public:
//! Init the hash
//! broker the index allocator
//! chunk_size the size of per chunk allocated, actual size may greater
//! factor factor = 1/ratio, ratio is the probability of a squence
//! number inserted to this container
//! max the max number key can be inserted
//! expansion_ratio memory expansion ratio
int init(HnswRabitqChunkBroker::Pointer &broker, uint32_t chunk_size,
uint32_t factor, size_t max, float expansion_ratio) {
ailego_assert_with(expansion_ratio > 1.0f, "ratio must > 1.0f");
broker_ = broker;
size_t items = std::ceil(chunk_size * 1.0f / sizeof(Item));
slot_items_ = 1UL << static_cast<size_t>((std::ceil(std::log2(items))));
size_t range = slot_items_ * factor / expansion_ratio;
mask_bits_ = std::floor(std::log2(range));
range = 1UL << mask_bits_;
size_t max_slots = std::ceil(max * 1.0f / range);
slots_.reserve(max_slots);
slot_loc_mask_ = slot_items_ - 1U;
int ret = load();
if (ret != 0) {
return ret;
}
LOG_DEBUG(
"HnswRabitqIndexHash init, chunkSize=%u factor=%u max=%zu "
"ratio=%f slotItems=%u maxSlots=%zu maskBits=%u "
"range=%zu",
chunk_size, factor, max, expansion_ratio, slot_items_, max_slots,
mask_bits_, range);
return 0;
}
int cleanup(void) {
broker_.reset();
slots_.clear();
slots_.shrink_to_fit();
mask_bits_ = 0U;
slot_items_ = 0U;
slot_loc_mask_ = 0U;
return 0;
}
const_iterator end(void) const {
return nullptr;
}
const_iterator find(const key_type key) const {
auto idx = key >> mask_bits_;
if (idx >= slots_.size()) {
return end();
}
auto it = slots_[idx].find(key, slot_items_, slot_loc_mask_);
return it && it->second != EmptyVal ? it : nullptr;
}
bool insert(key_type key, val_type val) {
auto idx = key >> mask_bits_;
if (idx >= slots_.size()) {
if (ailego_unlikely(idx >= slots_.capacity())) {
LOG_ERROR("no space to insert");
return false;
}
for (auto i = slots_.size(); i <= idx; ++i) {
if (ailego_unlikely(!alloc_slot(i))) {
return false;
}
}
}
auto it = slots_[idx].find(key, slot_items_, slot_loc_mask_);
if (ailego_unlikely(it == nullptr)) {
LOG_ERROR("no space to insert");
return false;
}
//! TODO: write memory is ok?
const_cast<iterator>(it)->first = key;
const_cast<iterator>(it)->second = val;
return slots_[idx].update(it);
}
private:
bool alloc_slot(size_t idx) {
ailego_assert_with(idx == slots_.size(), "invalid idx");
size_t size = slot_items_ * sizeof(Item);
auto p = broker_->alloc_chunk(
HnswRabitqChunkBroker::CHUNK_TYPE_NEIGHBOR_INDEX, idx, size);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return false;
}
Chunk::Pointer chunk = p.second;
if (ailego_unlikely(chunk->resize(size) != size)) {
LOG_ERROR("Chunk resize failed, size=%zu", size);
return false;
}
//! Read the whole data to memory
IndexStorage::MemoryBlock data_block;
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
LOG_ERROR("Chunk read failed, size=%zu", size);
return false;
}
slots_.emplace_back(std::move(chunk), std::move(data_block));
return true;
}
int load(void) {
size_t slots_cnt = broker_->get_chunk_cnt(
HnswRabitqChunkBroker::CHUNK_TYPE_NEIGHBOR_INDEX);
for (size_t i = 0UL; i < slots_cnt; ++i) {
auto chunk = broker_->get_chunk(
HnswRabitqChunkBroker::CHUNK_TYPE_NEIGHBOR_INDEX, i);
if (!chunk) {
LOG_ERROR("Get chunk failed, seq=%zu", i);
return IndexError_InvalidFormat;
}
size_t size = sizeof(Item) * slot_items_;
if (chunk->data_size() < size) {
LOG_ERROR(
"Hash params may be mismatch, seq=%zu, data_size=%zu "
"expect=%zu",
i, chunk->data_size(), size);
return IndexError_InvalidFormat;
}
//! Read the whole data to memory
IndexStorage::MemoryBlock data_block;
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
LOG_ERROR("Chunk read failed, size=%zu", size);
return false;
}
slots_.emplace_back(std::move(chunk), std::move(data_block));
}
return 0;
}
private:
HnswRabitqChunkBroker::Pointer broker_{}; // chunk broker
std::vector<Slot> slots_{};
uint32_t mask_bits_{0U};
uint32_t slot_items_{}; // must be a power of 2
uint32_t slot_loc_mask_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,134 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_searcher.h"
#include "zvec/core/framework/index_streamer.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
class HnswRabitqIndexProvider : public IndexProvider {
public:
HnswRabitqIndexProvider(const IndexMeta &meta,
const HnswRabitqEntity::Pointer &entity,
const std::string &owner)
: meta_(meta), entity_(entity), owner_class_(owner) {}
HnswRabitqIndexProvider(const HnswRabitqIndexProvider &) = delete;
HnswRabitqIndexProvider &operator=(const HnswRabitqIndexProvider &) = delete;
public: // holder interface
//! Create a new iterator
IndexProvider::Iterator::Pointer create_iterator() override {
return HnswRabitqIndexProvider::Iterator::Pointer(new (std::nothrow)
Iterator(entity_));
}
//! Retrieve count of vectors
size_t count(void) const override {
return entity_->doc_cnt();
}
//! Retrieve dimension of vector
size_t dimension(void) const override {
return meta_.dimension();
}
//! Retrieve type of vector
IndexMeta::DataType data_type(void) const override {
return meta_.data_type();
}
//! Retrieve vector size in bytes
size_t element_size(void) const override {
return meta_.element_size();
}
public: // provider's unique interface
//! Retrieve a vector using a primary key
const void *get_vector(uint64_t key) const override {
return entity_->get_vector_by_key(key);
}
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector_by_key(key, block);
}
//! Retrieve the owner class
const std::string &owner_class(void) const override {
return owner_class_;
}
private:
class Iterator : public IndexProvider::Iterator {
public:
Iterator(const HnswRabitqEntity::Pointer &entity)
: entity_(entity), cur_id_(0U) {}
//! Retrieve pointer of data
//! NOTICE: the vec feature will be changed after iterating to next, so
//! the caller need to keep a copy of it before iterator to next vector
virtual const void *data(void) const override {
return entity_->get_vector(cur_id_);
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
return cur_id_ < entity_->doc_cnt();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
return entity_->get_key(cur_id_);
}
//! Next iterator
virtual void next(void) override {
// cur_id_ += 1;
cur_id_ = get_next_valid_id(cur_id_ + 1);
}
//! Reset the iterator
void reset(void) {
cur_id_ = get_next_valid_id(0);
}
private:
node_id_t get_next_valid_id(node_id_t start_id) {
for (node_id_t i = start_id; i < entity_->doc_cnt(); i++) {
if (entity_->get_key(i) != kInvalidNodeId) {
cur_id_ = i;
return i;
}
}
return kInvalidNodeId;
}
private:
const HnswRabitqEntity::Pointer entity_;
node_id_t cur_id_;
};
private:
const IndexMeta &meta_;
const HnswRabitqEntity::Pointer entity_;
const std::string owner_class_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,121 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
namespace zvec {
namespace core {
inline const std::string PARAM_HNSW_RABITQ_GENERAL_DIMENSION(
"proxima.hnsw_rabitq.general.dimension");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_THREAD_COUNT(
"proxima.hnsw_rabitq.builder.thread_count");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_MEMORY_QUOTA(
"proxima.hnsw_rabitq.builder.memory_quota");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_EFCONSTRUCTION(
"proxima.hnsw_rabitq.builder.efconstruction");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_SCALING_FACTOR(
"proxima.hnsw_rabitq.builder.scaling_factor");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_CHECK_INTERVAL_SECS(
"proxima.hnsw_rabitq.builder.check_interval_secs");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_NEIGHBOR_PRUNE_MULTIPLIER(
"proxima.hnsw_rabitq.builder.neighbor_prune_multiplier");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_MIN_NEIGHBOR_COUNT(
"proxima.hnsw_rabitq.builder.min_neighbor_count");
inline const std::string PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT(
"proxima.hnsw_rabitq.builder.max_neighbor_count");
inline const std::string
PARAM_HNSW_RABITQ_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
"proxima.hnsw_rabitq.builder.l0_max_neighbor_count_multiplier");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_EF(
"proxima.hnsw_rabitq.searcher.ef");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD(
"proxima.hnsw_rabitq.searcher.brute_force_threshold");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_NEIGHBORS_IN_MEMORY_ENABLE(
"proxima.hnsw_rabitq.searcher.neighbors_in_memory_enable");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO(
"proxima.hnsw_rabitq.searcher.max_scan_ratio");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_CHECK_CRC_ENABLE(
"proxima.hnsw_rabitq.searcher.check_crc_enable");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_ENABLE(
"proxima.hnsw_rabitq.searcher.visit_bloomfilter_enable");
inline const std::string
PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB(
"proxima.hnsw_rabitq.searcher.visit_bloomfilter_negative_prob");
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_FORCE_PADDING_RESULT_ENABLE(
"proxima.hnsw_rabitq.searcher.force_padding_result_enable");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO(
"proxima.hnsw_rabitq.streamer.max_scan_ratio");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT(
"proxima.hnsw_rabitq.streamer.min_scan_limit");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT(
"proxima.hnsw_rabitq.streamer.max_scan_limit");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_EF(
"proxima.hnsw_rabitq.streamer.ef");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_EFCONSTRUCTION(
"proxima.hnsw_rabitq.streamer.efconstruction");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT(
"proxima.hnsw_rabitq.streamer.max_neighbor_count");
inline const std::string
PARAM_HNSW_RABITQ_STREAMER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
"proxima.hnsw_rabitq.streamer.l0_max_neighbor_count_multiplier");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_SCALING_FACTOR(
"proxima.hnsw_rabitq.streamer.scaling_factor");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD(
"proxima.hnsw_rabitq.streamer.brute_force_threshold");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT(
"proxima.hnsw_rabitq.streamer.docs_hard_limit");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT(
"proxima.hnsw_rabitq.streamer.docs_soft_limit");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_INDEX_SIZE(
"proxima.hnsw_rabitq.streamer.max_index_size");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_ENABLE(
"proxima.hnsw_rabitq.streamer.visit_bloomfilter_enable");
inline const std::string
PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB(
"proxima.hnsw_rabitq.streamer.visit_bloomfilter_negative_prob");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_CHECK_CRC_ENABLE(
"proxima.hnsw_rabitq.streamer.check_crc_enable");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_NEIGHBOR_PRUNE_MULTIPLIER(
"proxima.hnsw_rabitq.streamer.neighbor_prune_multiplier");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_CHUNK_SIZE(
"proxima.hnsw_rabitq.streamer.chunk_size");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_FILTER_SAME_KEY(
"proxima.hnsw_rabitq.streamer.filter_same_key");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_GET_VECTOR_ENABLE(
"proxima.hnsw_rabitq.streamer.get_vector_enable");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MIN_NEIGHBOR_COUNT(
"proxima.hnsw_rabitq.streamer.min_neighbor_count");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_FORCE_PADDING_RESULT_ENABLE(
"proxima.hnsw_rabitq.streamer.force_padding_result_enable");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_ESTIMATE_DOC_COUNT(
"proxima.hnsw_rabitq.streamer.estimate_doc_count");
inline const std::string PARAM_HNSW_RABITQ_STREAMER_USE_ID_MAP(
"proxima.hnsw_rabitq.streamer.use_id_map");
inline const std::string PARAM_HNSW_RABITQ_REDUCER_WORKING_PATH(
"proxima.hnsw_rabitq.reducer.working_path");
inline const std::string PARAM_HNSW_RABITQ_REDUCER_NUM_OF_ADD_THREADS(
"proxima.hnsw_rabitq.reducer.num_of_add_threads");
inline const std::string PARAM_HNSW_RABITQ_REDUCER_INDEX_NAME(
"proxima.hnsw_rabitq.reducer.index_name");
inline const std::string PARAM_HNSW_RABITQ_REDUCER_EFCONSTRUCTION(
"proxima.hnsw_rabitq.reducer.efconstruction");
} // namespace core
} // namespace zvec
@@ -0,0 +1,354 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_query_algorithm.h"
#include <chrono>
#include <iostream>
#include <ailego/internal/cpu_features.h>
#include <rabitqlib/index/estimator.hpp>
#include "zvec/ailego/internal/platform.h"
#include "hnsw_rabitq_entity.h"
#include "hnsw_rabitq_query_entity.h"
namespace zvec {
namespace core {
HnswRabitqQueryAlgorithm::HnswRabitqQueryAlgorithm(HnswRabitqEntity &entity,
size_t num_clusters,
RabitqMetricType metric_type)
: entity_(entity),
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
lock_pool_(kLockCnt),
num_clusters_(num_clusters),
metric_type_(metric_type) {
ex_bits_ = entity_.ex_bits();
padded_dim_ = entity_.padded_dim();
ip_func_ = rabitqlib::select_excode_ipfunc(ex_bits_);
LOG_INFO(
"Create query algorithm. num_clusters=%zu ex_bits=%zu padded_dim=%zu",
num_clusters_, ex_bits_, padded_dim_);
}
int HnswRabitqQueryAlgorithm::cleanup() {
return 0;
}
int HnswRabitqQueryAlgorithm::search(HnswRabitqQueryEntity *entity,
HnswRabitqContext *ctx) const {
spin_lock_.lock();
auto maxLevel = entity_.cur_max_level();
auto entry_point = entity_.entry_point();
spin_lock_.unlock();
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
return 0;
}
EstimateRecord curest;
get_bin_est(entity_.get_vector(entry_point), curest, *entity);
for (level_t cur_level = maxLevel; cur_level >= 1; --cur_level) {
select_entry_point(cur_level, &entry_point, &curest, ctx, entity);
}
auto &topk_heap = ctx->topk_heap();
topk_heap.clear();
search_neighbors(0, &entry_point, &curest, topk_heap, ctx, entity);
if (ctx->group_by_search()) {
expand_neighbors_by_group(topk_heap, ctx, entity);
}
return 0;
}
//! select_entry_point on hnsw level, ef = 1
void HnswRabitqQueryAlgorithm::select_entry_point(
level_t level, node_id_t *entry_point, EstimateRecord *curest,
HnswRabitqContext *ctx, HnswRabitqQueryEntity *query_entity) const {
auto &entity = ctx->get_entity();
while (true) {
const Neighbors neighbors = entity.get_neighbors(level, *entry_point);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
ailego_prefetch(neighbors.data);
uint32_t size = neighbors.size();
if (size == 0) {
break;
}
bool find_closer = false;
for (uint32_t i = 0; i < size; ++i) {
EstimateRecord candest;
get_bin_est(entity_.get_vector(neighbors[i]), candest, *query_entity);
if (candest.est_dist < curest->est_dist) {
*curest = candest;
*entry_point = neighbors[i];
find_closer = true;
}
}
if (!find_closer) {
break;
}
}
return;
}
void HnswRabitqQueryAlgorithm::search_neighbors(
level_t level, node_id_t *entry_point, EstimateRecord *dist, TopkHeap &topk,
HnswRabitqContext *ctx, HnswRabitqQueryEntity *query_entity) const {
const auto &entity = ctx->get_entity();
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (ctx->filter().is_valid()) {
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
}
candidates.clear();
visit.clear();
visit.set_visited(*entry_point);
if (!filter(*entry_point)) {
topk.emplace(*entry_point, ResultRecord(*dist));
}
candidates.emplace(*entry_point, ResultRecord(*dist));
while (!candidates.empty() && !ctx->reach_scan_limit()) {
auto top = candidates.begin();
node_id_t main_node = top->first;
auto main_dist = top->second;
if (topk.full() && main_dist.est_dist > topk[0].second.est_dist) {
break;
}
candidates.pop();
const Neighbors neighbors = entity.get_neighbors(level, main_node);
ailego_prefetch(neighbors.data);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) {
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_visit_dup_cnt())++;
}
continue;
}
visit.set_visited(node);
neighbor_ids[size++] = node;
}
if (size == 0) {
continue;
}
for (uint32_t i = 0; i < size; ++i) {
node_id_t node = neighbor_ids[i];
EstimateRecord candest;
auto *cand_vector = entity_.get_vector(node);
ailego_prefetch(cand_vector);
get_bin_est(cand_vector, candest, *query_entity);
if (ex_bits_ > 0) {
// Check preliminary score against current worst full estimate.
bool flag_update_KNNs =
(!topk.full()) || candest.low_dist < topk[0].second.est_dist;
if (flag_update_KNNs) {
// Compute the full estimate if promising.
get_full_est(cand_vector, candest, *query_entity);
} else {
continue;
}
} else {
// ex_bits_ == 0: est_dist is already the best estimate
if (topk.full() && candest.est_dist >= topk[0].second.est_dist) {
continue;
}
}
candidates.emplace(node, ResultRecord(candest));
// update entry_point for next level scan
if (candest < *dist) {
*entry_point = node;
*dist = candest;
}
if (!filter(node)) {
topk.emplace(node, ResultRecord(candest));
}
} // end for
} // while
return;
}
void HnswRabitqQueryAlgorithm::expand_neighbors_by_group(
TopkHeap &topk, HnswRabitqContext *ctx,
HnswRabitqQueryEntity *query_entity) const {
if (!ctx->group_by().is_valid()) {
return;
}
const auto &entity = ctx->get_entity();
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity.get_key(id));
};
// devide into groups
std::map<std::string, TopkHeap> &group_topk_heaps = ctx->group_topk_heaps();
for (uint32_t i = 0; i < topk.size(); ++i) {
node_id_t id = topk[i].first;
auto score = topk[i].second;
std::string group_id = group_by(id);
auto &topk_heap = group_topk_heaps[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, score);
}
// stage 2, expand to reach group num as possible
if (group_topk_heaps.size() < ctx->group_num()) {
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (ctx->filter().is_valid()) {
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
}
// refill to get enough groups
candidates.clear();
visit.clear();
for (uint32_t i = 0; i < topk.size(); ++i) {
node_id_t id = topk[i].first;
auto score = topk[i].second;
visit.set_visited(id);
candidates.emplace_back(id, score);
}
// do expand
while (!candidates.empty() && !ctx->reach_scan_limit()) {
auto top = candidates.begin();
node_id_t main_node = top->first;
candidates.pop();
const Neighbors neighbors = entity.get_neighbors(0, main_node);
ailego_prefetch(neighbors.data);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) {
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_visit_dup_cnt())++;
}
continue;
}
visit.set_visited(node);
neighbor_ids[size++] = node;
}
if (size == 0) {
continue;
}
for (uint32_t i = 0; i < size; ++i) {
node_id_t node = neighbor_ids[i];
EstimateRecord candest;
auto *cand_vector = entity_.get_vector(node);
ailego_prefetch(cand_vector);
get_full_est(cand_vector, candest, *query_entity);
if (!filter(node)) {
std::string group_id = group_by(node);
auto &topk_heap = group_topk_heaps[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(node, ResultRecord(candest));
if (group_topk_heaps.size() >= ctx->group_num()) {
break;
}
}
candidates.emplace(node, ResultRecord(candest));
} // end for
} // end while
} // end if
}
void HnswRabitqQueryAlgorithm::get_bin_est(
const void *vector, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const {
const auto &q_to_centroids = entity.q_to_centroids;
auto &query_wrapper = *entity.query_wrapper;
uint32_t cluster_id = entity_.get_cluster_id(vector);
const char *bin_data = entity_.get_bin_data(vector);
if (metric_type_ == RabitqMetricType::kIP) {
float norm = q_to_centroids[cluster_id];
float error = q_to_centroids[cluster_id + num_clusters_];
rabitqlib::split_single_estdist(bin_data, query_wrapper, padded_dim_,
res.ip_x0_qr, res.est_dist, res.low_dist,
-norm, error);
} else {
// L2 distance
float norm = q_to_centroids[cluster_id];
rabitqlib::split_single_estdist(bin_data, query_wrapper, padded_dim_,
res.ip_x0_qr, res.est_dist, res.low_dist,
norm * norm, norm);
}
}
void HnswRabitqQueryAlgorithm::get_full_est(
const void *vector, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const {
const auto &q_to_centroids = entity.q_to_centroids;
auto &query_wrapper = *entity.query_wrapper;
uint32_t cluster_id = entity_.get_cluster_id(vector);
const char *bin_data = entity_.get_bin_data(vector);
const char *ex_data = entity_.get_ex_data(vector);
if (metric_type_ == RabitqMetricType::kIP) {
float norm = q_to_centroids[cluster_id];
float error = q_to_centroids[cluster_id + num_clusters_];
rabitqlib::split_single_fulldist(bin_data, ex_data, ip_func_, query_wrapper,
padded_dim_, ex_bits_, res.est_dist,
res.low_dist, res.ip_x0_qr, -norm, error);
} else {
// L2 distance
float norm = q_to_centroids[cluster_id];
rabitqlib::split_single_fulldist(
bin_data, ex_data, ip_func_, query_wrapper, padded_dim_, ex_bits_,
res.est_dist, res.low_dist, res.ip_x0_qr, norm * norm, norm);
}
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,136 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
#include <ailego/parallel/lock.h>
#include "hnsw_rabitq_context.h"
#include "hnsw_rabitq_dist_calculator.h"
#include "hnsw_rabitq_entity.h"
#include "rabitq_params.h"
namespace zvec {
namespace core {
class HnswRabitqQueryEntity;
//! hnsw graph algorithm implement
class HnswRabitqQueryAlgorithm {
public:
typedef std::unique_ptr<HnswRabitqQueryAlgorithm> UPointer;
public:
//! Constructor
explicit HnswRabitqQueryAlgorithm(HnswRabitqEntity &entity,
size_t num_clusters,
RabitqMetricType metric_type);
//! Destructor
~HnswRabitqQueryAlgorithm() = default;
//! Cleanup HnswRabitqQueryAlgorithm
int cleanup();
//! do knn search in graph
//! return 0 on success, or errCode in failure. results saved in ctx
int search(HnswRabitqQueryEntity *entity, HnswRabitqContext *ctx) const;
//! Initiate HnswRabitqQueryAlgorithm
int init() {
level_probas_.clear();
double level_mult =
1 / std::log(static_cast<double>(entity_.scaling_factor()));
for (int level = 0;; level++) {
// refers faiss get_random_level alg
double proba =
std::exp(-level / level_mult) * (1 - std::exp(-1 / level_mult));
if (proba < 1e-9) {
break;
}
level_probas_.push_back(proba);
}
return 0;
}
//! Generate a random level
//! return graph level
uint32_t get_random_level() const {
// gen rand float (0, 1)
double f = mt_() / static_cast<float>(mt_.max());
for (size_t level = 0; level < level_probas_.size(); level++) {
if (f < level_probas_[level]) {
return level;
}
f -= level_probas_[level];
}
return level_probas_.size() - 1;
}
void get_full_est(node_id_t id, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const {
return get_full_est(entity_.get_vector(id), res, entity);
}
private:
//! Select in upper layer to get entry point for next layer search
void select_entry_point(level_t level, node_id_t *entry_point,
EstimateRecord *dist, HnswRabitqContext *ctx,
HnswRabitqQueryEntity *entity) const;
//! Given a node id and level, search the nearest neighbors in graph
//! Note: the nearest neighbors result keeps in topk, and entry_point and
//! dist will be updated to current level nearest node id and distance
void search_neighbors(level_t level, node_id_t *entry_point,
EstimateRecord *dist, TopkHeap &topk,
HnswRabitqContext *ctx,
HnswRabitqQueryEntity *entity) const;
//! expand neighbors until group nums are reached
void expand_neighbors_by_group(TopkHeap &topk, HnswRabitqContext *ctx,
HnswRabitqQueryEntity *query_entity) const;
void get_full_est(const void *vector, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const;
void get_bin_est(const void *vector, EstimateRecord &res,
HnswRabitqQueryEntity &entity) const;
private:
HnswRabitqQueryAlgorithm(const HnswRabitqQueryAlgorithm &) = delete;
HnswRabitqQueryAlgorithm &operator=(const HnswRabitqQueryAlgorithm &) =
delete;
private:
static constexpr uint32_t kLockCnt{1U << 8};
static constexpr uint32_t kLockMask{kLockCnt - 1U};
HnswRabitqEntity &entity_;
mutable std::mt19937 mt_{};
std::vector<double> level_probas_{};
mutable ailego::SpinMutex spin_lock_{}; // global spin lock
std::mutex mutex_{}; // global mutex
// TODO: spin lock?
std::vector<std::mutex> lock_pool_{};
size_t num_clusters_{0};
RabitqMetricType metric_type_{RabitqMetricType::kL2};
size_t padded_dim_{0};
size_t ex_bits_{0};
float (*ip_func_)(const float *, const uint8_t *, size_t);
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,28 @@
// Copyright 2025-present the centaurdb project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
#pragma once
#include <memory>
#include <vector>
#include <rabitqlib/index/query.hpp>
namespace zvec::core {
struct HnswRabitqQueryEntity {
std::vector<float> rotated_query;
std::vector<float> q_to_centroids;
std::unique_ptr<rabitqlib::SplitSingleQuery<float>> query_wrapper;
};
} // namespace zvec::core
@@ -0,0 +1,28 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
#include "hnsw_rabitq_builder.h"
#include "hnsw_rabitq_searcher.h"
#include "hnsw_rabitq_streamer.h"
#include "rabitq_converter.h"
#include "rabitq_reformer.h"
namespace zvec::core {
INDEX_FACTORY_REGISTER_STREAMER(HnswRabitqStreamer);
INDEX_FACTORY_REGISTER_REFORMER_ALIAS(RabitqReformer, RabitqReformer);
INDEX_FACTORY_REGISTER_SEARCHER(HnswRabitqSearcher);
INDEX_FACTORY_REGISTER_CONVERTER_ALIAS(RabitqConverter, RabitqConverter);
INDEX_FACTORY_REGISTER_BUILDER(HnswRabitqBuilder);
} // namespace zvec::core
@@ -0,0 +1,516 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_searcher.h"
#include <rabitqlib/utils/rotator.hpp>
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_entity.h"
#include "hnsw_rabitq_index_provider.h"
#include "hnsw_rabitq_params.h"
#include "hnsw_rabitq_query_entity.h"
#include "hnsw_rabitq_searcher_entity.h"
#include "rabitq_params.h"
namespace zvec {
namespace core {
HnswRabitqSearcher::HnswRabitqSearcher() {}
HnswRabitqSearcher::~HnswRabitqSearcher() {}
int HnswRabitqSearcher::init(const ailego::Params &search_params) {
params_ = search_params;
params_.get(PARAM_HNSW_RABITQ_SEARCHER_EF, &ef_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO, &max_scan_ratio_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_ENABLE,
&bf_enabled_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_CHECK_CRC_ENABLE, &check_crc_enabled_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_NEIGHBORS_IN_MEMORY_ENABLE,
&neighbors_in_memory_enabled_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB,
&bf_negative_probability_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
params_.get(PARAM_HNSW_RABITQ_SEARCHER_FORCE_PADDING_RESULT_ENABLE,
&force_padding_topk_enabled_);
if (ef_ == 0) {
ef_ = HnswRabitqEntity::kDefaultEf;
}
if (bf_negative_probability_ <= 0.0f || bf_negative_probability_ >= 1.0f) {
LOG_ERROR(
"[%s] must be in range (0,1)",
PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB.c_str());
return IndexError_InvalidArgument;
}
entity_.set_neighbors_in_memory(neighbors_in_memory_enabled_);
ailego::Params reformer_params;
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
int ret = reformer_.init(reformer_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
return ret;
}
state_ = STATE_INITED;
LOG_DEBUG(
"Init params: ef=%u maxScanRatio=%f bfEnabled=%u checkCrcEnabled=%u "
"neighborsInMemoryEnabled=%u bfNagtiveProb=%f bruteForceThreshold=%u "
"forcePadding=%u",
ef_, max_scan_ratio_, bf_enabled_, check_crc_enabled_,
neighbors_in_memory_enabled_, bf_negative_probability_,
bruteforce_threshold_, force_padding_topk_enabled_);
return 0;
}
void HnswRabitqSearcher::print_debug_info() {
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
Neighbors neighbours = entity_.get_neighbors(0, id);
std::cout << "node: " << id << "; ";
for (uint32_t i = 0; i < neighbours.size(); ++i) {
std::cout << neighbours[i];
if (i == neighbours.size() - 1) {
std::cout << std::endl;
} else {
std::cout << ", ";
}
}
}
}
int HnswRabitqSearcher::cleanup() {
LOG_INFO("Begin HnswRabitqSearcher:cleanup");
metric_.reset();
meta_.clear();
stats_.clear_attributes();
stats_.set_loaded_count(0UL);
stats_.set_loaded_costtime(0UL);
max_scan_ratio_ = HnswRabitqEntity::kDefaultScanRatio;
max_scan_num_ = 0U;
ef_ = HnswRabitqEntity::kDefaultEf;
bf_enabled_ = false;
bf_negative_probability_ = HnswRabitqEntity::kDefaultBFNegativeProbability;
bruteforce_threshold_ = HnswRabitqEntity::kDefaultBruteForceThreshold;
check_crc_enabled_ = false;
neighbors_in_memory_enabled_ = false;
entity_.cleanup();
state_ = STATE_INIT;
LOG_INFO("End HnswRabitqSearcher:cleanup");
return 0;
}
int HnswRabitqSearcher::load(IndexStorage::Pointer container,
IndexMetric::Pointer metric) {
if (state_ != STATE_INITED) {
LOG_ERROR("Init the searcher first before load index");
return IndexError_Runtime;
}
LOG_INFO("Begin HnswRabitqSearcher:load");
auto start_time = ailego::Monotime::MilliSeconds();
int ret = IndexHelper::DeserializeFromStorage(container.get(), &meta_);
if (ret != 0) {
LOG_ERROR("Failed to deserialize meta from container");
return ret;
}
ret = reformer_.load(container);
if (ret != 0) {
LOG_ERROR("Failed to load reformer from container: %d", ret);
return ret;
}
ret = entity_.load(container, check_crc_enabled_);
if (ret != 0) {
LOG_ERROR("HnswRabitqSearcher load index failed");
return ret;
}
alg_ = HnswRabitqQueryAlgorithm::UPointer(new HnswRabitqQueryAlgorithm(
entity_, reformer_.num_clusters(), reformer_.rabitq_metric_type()));
if (metric) {
metric_ = metric;
} else {
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!metric_) {
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
return IndexError_NoExist;
}
ret = metric_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
return ret;
}
if (metric_->query_metric()) {
metric_ = metric_->query_metric();
}
}
if (!metric_->is_matched(meta_)) {
LOG_ERROR("IndexMetric not match index meta");
return IndexError_Mismatch;
}
max_scan_num_ = static_cast<uint32_t>(max_scan_ratio_ * entity_.doc_cnt());
max_scan_num_ = std::max(4096U, max_scan_num_);
stats_.set_loaded_count(entity_.doc_cnt());
stats_.set_loaded_costtime(ailego::Monotime::MilliSeconds() - start_time);
state_ = STATE_LOADED;
magic_ = IndexContext::GenerateMagic();
LOG_INFO("End HnswRabitqSearcher::load");
return 0;
}
int HnswRabitqSearcher::unload() {
LOG_INFO("HnswRabitqSearcher unload index");
meta_.clear();
entity_.cleanup();
metric_.reset();
max_scan_num_ = 0;
stats_.set_loaded_count(0UL);
stats_.set_loaded_costtime(0UL);
state_ = STATE_INITED;
return 0;
}
int HnswRabitqSearcher::update_context(HnswRabitqContext *ctx) const {
const HnswRabitqEntity::Pointer entity = entity_.clone();
if (!entity) {
LOG_ERROR("Failed to clone search context entity");
return IndexError_Runtime;
}
ctx->set_max_scan_num(max_scan_num_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
return ctx->update_context(HnswRabitqContext::kSearcherContext, meta_,
metric_, entity, magic_);
}
int HnswRabitqSearcher::search_impl(const void *query,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (entity_.doc_cnt() <= ctx->get_bruteforce_threshold()) {
return search_bf_impl(query, qmeta, count, context);
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
int ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->resize_results(count);
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ret = alg_->search(&entity, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher fast search failed");
return ret;
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int HnswRabitqSearcher::search_bf_impl(const void *query,
const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
int ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->resize_results(count);
if (ctx->group_by_search()) {
if (!ctx->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
};
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ctx->group_topk_heaps().clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
EstimateRecord dist;
alg_->get_full_est(id, dist, entity);
std::string group_id = group_by(id);
auto &topk_heap = ctx->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
} else {
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ctx->topk_heap().clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
EstimateRecord dist;
alg_->get_full_est(id, dist, entity);
ctx->topk_heap().emplace(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int HnswRabitqSearcher::search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
if (ailego_unlikely(!query || !context)) {
LOG_ERROR("The context is not created by this searcher");
return IndexError_Mismatch;
}
if (ailego_unlikely(p_keys.size() != count)) {
LOG_ERROR("The size of p_keys is not equal to count");
return IndexError_InvalidArgument;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
int ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->resize_results(count);
if (ctx->group_by_search()) {
if (!ctx->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
};
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ctx->group_topk_heaps().clear();
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
uint64_t pk = p_keys[q][idx];
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
node_id_t id = entity_.get_id(pk);
if (id != kInvalidNodeId) {
EstimateRecord dist;
alg_->get_full_est(id, dist, entity);
std::string group_id = group_by(id);
auto &topk_heap = ctx->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, dist);
}
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
} else {
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
int ret = reformer_.transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ctx->topk_heap().clear();
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
uint64_t pk = p_keys[q][idx];
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
node_id_t id = entity_.get_id(pk);
if (id != kInvalidNodeId) {
EstimateRecord dist;
alg_->get_full_est(id, dist, entity);
ctx->topk_heap().emplace(id, dist);
}
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
IndexSearcher::Context::Pointer HnswRabitqSearcher::create_context() const {
if (ailego_unlikely(state_ != STATE_LOADED)) {
LOG_ERROR("Load the index first before create context");
return Context::Pointer();
}
const HnswRabitqEntity::Pointer search_ctx_entity = entity_.clone();
if (!search_ctx_entity) {
LOG_ERROR("Failed to create search context entity");
return Context::Pointer();
}
HnswRabitqContext *ctx = new (std::nothrow)
HnswRabitqContext(meta_.dimension(), metric_, search_ctx_entity);
if (ailego_unlikely(ctx == nullptr)) {
LOG_ERROR("Failed to new HnswRabitqContext");
return Context::Pointer();
}
ctx->set_ef(ef_);
ctx->set_max_scan_num(max_scan_num_);
uint32_t filter_mode =
bf_enabled_ ? VisitFilter::BloomFilter : VisitFilter::ByteMap;
ctx->set_filter_mode(filter_mode);
ctx->set_filter_negative_probability(bf_negative_probability_);
ctx->set_magic(magic_);
ctx->set_force_padding_topk(force_padding_topk_enabled_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
if (ailego_unlikely(ctx->init(HnswRabitqContext::kSearcherContext)) != 0) {
LOG_ERROR("Init HnswRabitqContext failed");
delete ctx;
return Context::Pointer();
}
return Context::Pointer(ctx);
}
IndexProvider::Pointer HnswRabitqSearcher::create_provider(void) const {
LOG_DEBUG("HnswRabitqSearcher create provider");
auto entity = entity_.clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("Clone HnswRabitqEntity failed");
return Provider::Pointer();
}
return Provider::Pointer(new (std::nothrow) HnswRabitqIndexProvider(
meta_, entity, "HnswRabitqSearcher"));
}
const void *HnswRabitqSearcher::get_vector(uint64_t key) const {
return entity_.get_vector_by_key(key);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,142 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "zvec/core/framework/index_framework.h"
#include "hnsw_rabitq_query_algorithm.h"
#include "hnsw_rabitq_searcher_entity.h"
#include "rabitq_reformer.h"
namespace zvec {
namespace core {
class HnswRabitqSearcher : public IndexSearcher {
public:
using ContextPointer = IndexSearcher::Context::Pointer;
public:
HnswRabitqSearcher(void);
~HnswRabitqSearcher(void);
HnswRabitqSearcher(const HnswRabitqSearcher &) = delete;
HnswRabitqSearcher &operator=(const HnswRabitqSearcher &) = delete;
protected:
//! Initialize Searcher
virtual int init(const ailego::Params &params) override;
//! Cleanup Searcher
virtual int cleanup(void) override;
//! Load Index from storage
virtual int load(IndexStorage::Pointer container,
IndexMetric::Pointer metric) override;
//! Unload index from storage
virtual int unload(void) override;
//! KNN Search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_impl(query, qmeta, 1, context);
}
//! KNN Search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
ContextPointer &context) const override;
//! Linear Search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_impl(query, qmeta, 1, context);
}
//! Linear Search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
ContextPointer &context) const override;
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
//! Fetch vector by key
virtual const void *get_vector(uint64_t key) const override;
//! Create a searcher context
virtual ContextPointer create_context() const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
return meta_;
}
//! Retrieve params of index
virtual const ailego::Params &params(void) const override {
return params_;
}
virtual void print_debug_info() override;
private:
//! To share ctx across streamer/searcher, we need to update the context for
//! current streamer/searcher
int update_context(HnswRabitqContext *ctx) const;
private:
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
HnswRabitqSearcherEntity entity_{};
HnswRabitqQueryAlgorithm::UPointer alg_; // impl graph algorithm
IndexMetric::Pointer metric_{};
IndexMeta meta_{};
ailego::Params params_{};
Stats stats_;
uint32_t ef_{HnswRabitqEntity::kDefaultEf};
uint32_t max_scan_num_{0U};
uint32_t bruteforce_threshold_{HnswRabitqEntity::kDefaultBruteForceThreshold};
float max_scan_ratio_{HnswRabitqEntity::kDefaultScanRatio};
bool bf_enabled_{false};
bool check_crc_enabled_{false};
bool neighbors_in_memory_enabled_{false};
bool force_padding_topk_enabled_{false};
float bf_negative_probability_{
HnswRabitqEntity::kDefaultBFNegativeProbability};
uint32_t magic_{0U};
RabitqReformer reformer_;
State state_{STATE_INIT};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,515 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_searcher_entity.h"
#include <zvec/ailego/hash/crc32c.h>
#include "utility/sparse_utility.h"
namespace zvec {
namespace core {
HnswRabitqSearcherEntity::HnswRabitqSearcherEntity() {}
int HnswRabitqSearcherEntity::cleanup(void) {
storage_.reset();
vectors_.reset();
keys_.reset();
neighbors_.reset();
neighbors_meta_.reset();
neighbors_in_memory_enabled_ = false;
loaded_ = false;
this->HnswRabitqEntity::cleanup();
return 0;
}
key_t HnswRabitqSearcherEntity::get_key(node_id_t id) const {
const void *key;
if (ailego_unlikely(keys_->read(id * sizeof(key_t), &key, sizeof(key_t)) !=
sizeof(key_t))) {
LOG_ERROR("Read key from segment failed");
return kInvalidKey;
}
return *(reinterpret_cast<const key_t *>(key));
}
//! Get vector local id by key
node_id_t HnswRabitqSearcherEntity::get_id(key_t key) const {
if (ailego_unlikely(!mapping_)) {
LOG_ERROR("Index missing mapping segment");
return kInvalidNodeId;
}
//! Do binary search
node_id_t start = 0UL;
node_id_t end = doc_cnt();
const void *data;
node_id_t idx = 0u;
while (start < end) {
idx = start + (end - start) / 2;
if (ailego_unlikely(
mapping_->read(idx * sizeof(node_id_t), &data, sizeof(node_id_t)) !=
sizeof(node_id_t))) {
LOG_ERROR("Read key from segment failed");
return kInvalidNodeId;
}
const key_t *mkey;
node_id_t local_id = *reinterpret_cast<const node_id_t *>(data);
if (ailego_unlikely(keys_->read(local_id * sizeof(key_t),
(const void **)(&mkey),
sizeof(key_t)) != sizeof(key_t))) {
LOG_ERROR("Read key from segment failed");
return kInvalidNodeId;
}
if (*mkey < key) {
start = idx + 1;
} else if (*mkey > key) {
end = idx;
} else {
return local_id;
}
}
return kInvalidNodeId;
}
const void *HnswRabitqSearcherEntity::get_vector_by_key(key_t key) const {
node_id_t local_id = get_id(key);
if (ailego_unlikely(local_id == kInvalidNodeId)) {
return nullptr;
}
return get_vector(local_id);
}
const void *HnswRabitqSearcherEntity::get_vector(node_id_t id) const {
size_t read_size = vector_size();
size_t offset = node_size() * id;
const void *vec;
if (ailego_unlikely(vectors_->read(offset, &vec, read_size) != read_size)) {
LOG_ERROR("Read vector from segment failed");
return nullptr;
}
return vec;
}
int HnswRabitqSearcherEntity::get_vector(
const node_id_t id, IndexStorage::MemoryBlock &block) const {
const void *vec = get_vector(id);
block.reset((void *)vec);
return 0;
}
const void *HnswRabitqSearcherEntity::get_vectors() const {
const void *vec;
size_t len = node_size() * doc_cnt();
if (vectors_->read(0, &vec, len) != len) {
LOG_ERROR("Read vectors from segment failed");
return nullptr;
}
return vec;
}
int HnswRabitqSearcherEntity::get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const {
ailego_assert_with(count <= segment_datas_.size(), "invalid count");
size_t read_size = vector_size();
for (uint32_t i = 0; i < count; ++i) {
segment_datas_[i].offset = node_size() * ids[i];
segment_datas_[i].length = read_size;
ailego_assert_with(segment_datas_[i].offset < vectors_->data_size(),
"invalid offset");
}
if (ailego_unlikely(!vectors_->read(&segment_datas_[0], count))) {
LOG_ERROR("Read vectors from segment failed");
return IndexError_ReadData;
}
for (uint32_t i = 0; i < count; ++i) {
vecs[i] = segment_datas_[i].data;
}
return 0;
}
int HnswRabitqSearcherEntity::get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
const void *vecs[count];
get_vector(ids, count, vecs);
for (uint32_t i = 0; i < count; ++i) {
vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i]));
}
return 0;
}
const Neighbors HnswRabitqSearcherEntity::get_neighbors(level_t level,
node_id_t id) const {
if (level == 0) {
if (neighbors_in_memory_enabled_) {
auto hd = reinterpret_cast<const NeighborsHeader *>(
fixed_neighbors_.get() + neighbors_size() * id);
return {hd->neighbor_cnt, hd->neighbors};
}
const GraphNeighborMeta *m;
if (ailego_unlikely(neighbors_meta_->read(id * sizeof(GraphNeighborMeta),
(const void **)(&m),
sizeof(GraphNeighborMeta)) !=
sizeof(GraphNeighborMeta))) {
LOG_ERROR("Read neighbors meta from segment failed");
return {0, nullptr};
}
const void *data;
if (ailego_unlikely(neighbors_->read(m->offset, &data,
m->neighbor_cnt * sizeof(node_id_t)) !=
m->neighbor_cnt * sizeof(node_id_t))) {
LOG_ERROR("Read neighbors from segment failed");
return {0, nullptr};
}
return {static_cast<uint32_t>(m->neighbor_cnt),
reinterpret_cast<const node_id_t *>(data)};
}
//! Read level > 0 neighbors
const HnswNeighborMeta *m;
if (ailego_unlikely(upper_neighbors_meta_->read(id * sizeof(HnswNeighborMeta),
(const void **)(&m),
sizeof(HnswNeighborMeta)) !=
sizeof(HnswNeighborMeta))) {
LOG_ERROR("Read neighbors meta from segment failed");
return {0, nullptr};
}
ailego_assert_with(level <= m->level, "invalid level");
size_t offset = m->offset + (level - 1) * upper_neighbors_size();
ailego_assert_with(offset <= upper_neighbors_->data_size(), "invalid offset");
const void *data;
if (ailego_unlikely(
upper_neighbors_->read(offset, &data, upper_neighbors_size()) !=
upper_neighbors_size())) {
LOG_ERROR("Read neighbors from segment failed");
return {0, nullptr};
}
auto hd = reinterpret_cast<const NeighborsHeader *>(data);
return {hd->neighbor_cnt, hd->neighbors};
}
int HnswRabitqSearcherEntity::load(const IndexStorage::Pointer &container,
bool check_crc) {
storage_ = container;
int ret = load_segments(check_crc);
if (ret != 0) {
return ret;
}
loaded_ = true;
LOG_INFO(
"Index info: docCnt=%u entryPoint=%u maxLevel=%d efConstruct=%zu "
"l0NeighborCnt=%zu upperNeighborCnt=%zu scalingFactor=%zu "
"vectorSize=%zu nodeSize=%zu vectorSegmentSize=%zu keySegmentSize=%zu "
"neighborsSegmentSize=%zu neighborsMetaSegmentSize=%zu ",
doc_cnt(), entry_point(), cur_max_level(), ef_construction(),
l0_neighbor_cnt(), upper_neighbor_cnt(), scaling_factor(), vector_size(),
node_size(), vectors_->data_size(), keys_->data_size(),
neighbors_ == nullptr ? 0 : neighbors_->data_size(),
neighbors_meta_ == nullptr ? 0 : neighbors_meta_->data_size());
return 0;
}
int HnswRabitqSearcherEntity::load_segments(bool check_crc) {
//! load header
const void *data = nullptr;
HNSWHeader hd;
auto graph_hd_segment = storage_->get(kGraphHeaderSegmentId);
if (!graph_hd_segment || graph_hd_segment->data_size() < sizeof(hd.graph)) {
LOG_ERROR("Miss or invalid segment %s", kGraphHeaderSegmentId.c_str());
return IndexError_InvalidFormat;
}
if (graph_hd_segment->read(0, reinterpret_cast<const void **>(&data),
sizeof(hd.graph)) != sizeof(hd.graph)) {
LOG_ERROR("Read segment %s failed", kGraphHeaderSegmentId.c_str());
return IndexError_ReadData;
}
memcpy(&hd.graph, data, sizeof(hd.graph));
auto hnsw_hd_segment = storage_->get(kHnswHeaderSegmentId);
if (!hnsw_hd_segment || hnsw_hd_segment->data_size() < sizeof(hd.hnsw)) {
LOG_ERROR("Miss or invalid segment %s", kHnswHeaderSegmentId.c_str());
return IndexError_InvalidFormat;
}
if (hnsw_hd_segment->read(0, reinterpret_cast<const void **>(&data),
sizeof(hd.hnsw)) != sizeof(hd.hnsw)) {
LOG_ERROR("Read segment %s failed", kHnswHeaderSegmentId.c_str());
return IndexError_ReadData;
}
memcpy(&hd.hnsw, data, sizeof(hd.hnsw));
*mutable_header() = hd;
segment_datas_.resize(std::max(l0_neighbor_cnt(), upper_neighbor_cnt()));
vectors_ = storage_->get(kGraphFeaturesSegmentId);
if (!vectors_) {
LOG_ERROR("IndexStorage get segment %s failed",
kGraphFeaturesSegmentId.c_str());
return IndexError_InvalidFormat;
}
keys_ = storage_->get(kGraphKeysSegmentId);
if (!keys_) {
LOG_ERROR("IndexStorage get segment %s failed",
kGraphKeysSegmentId.c_str());
return IndexError_InvalidFormat;
}
neighbors_ = storage_->get(kGraphNeighborsSegmentId);
if (!neighbors_ || (neighbors_->data_size() == 0 && doc_cnt() > 1)) {
LOG_ERROR("IndexStorage get segment %s failed or empty",
kGraphNeighborsSegmentId.c_str());
return IndexError_InvalidArgument;
}
neighbors_meta_ = storage_->get(kGraphOffsetsSegmentId);
if (!neighbors_meta_ ||
neighbors_meta_->data_size() < sizeof(GraphNeighborMeta) * doc_cnt()) {
LOG_ERROR("IndexStorage get segment %s failed or invalid size",
kGraphOffsetsSegmentId.c_str());
return IndexError_InvalidArgument;
}
upper_neighbors_ = storage_->get(kHnswNeighborsSegmentId);
if (!upper_neighbors_ ||
(upper_neighbors_->data_size() == 0 && cur_max_level() > 0)) {
LOG_ERROR("IndexStorage get segment %s failed or empty",
kHnswNeighborsSegmentId.c_str());
return IndexError_InvalidArgument;
}
upper_neighbors_meta_ = storage_->get(kHnswOffsetsSegmentId);
if (!upper_neighbors_meta_ || upper_neighbors_meta_->data_size() <
sizeof(HnswNeighborMeta) * doc_cnt()) {
LOG_ERROR("IndexStorage get segment %s failed or invalid size",
kHnswOffsetsSegmentId.c_str());
return IndexError_InvalidArgument;
}
mapping_ = storage_->get(kGraphMappingSegmentId);
if (!mapping_ || mapping_->data_size() < sizeof(node_id_t) * doc_cnt()) {
LOG_ERROR("IndexStorage get segment %s failed or invalid size",
kGraphMappingSegmentId.c_str());
return IndexError_InvalidArgument;
}
if (check_crc) {
std::vector<SegmentPointer> segments;
segments.emplace_back(graph_hd_segment);
segments.emplace_back(hnsw_hd_segment);
segments.emplace_back(vectors_);
segments.emplace_back(keys_);
segments.emplace_back(neighbors_);
segments.emplace_back(neighbors_meta_);
segments.emplace_back(upper_neighbors_);
segments.emplace_back(upper_neighbors_meta_);
if (!do_crc_check(segments)) {
LOG_ERROR("Check index crc failed, the index may broken");
return IndexError_Runtime;
}
}
if (neighbors_in_memory_enabled_) {
int ret = load_and_flat_neighbors();
if (ret != 0) {
return ret;
}
}
return 0;
}
int HnswRabitqSearcherEntity::load_and_flat_neighbors() {
fixed_neighbors_.reset(
new (std::nothrow) char[neighbors_size() * doc_cnt()]{},
std::default_delete<char[]>());
if (!fixed_neighbors_) {
LOG_ERROR("Malloc memory failed");
return IndexError_NoMemory;
}
//! Get a new segemnt to release the buffer after loading neighbors
auto neighbors_meta = storage_->get(kGraphOffsetsSegmentId);
if (!neighbors_meta) {
LOG_ERROR("IndexStorage get segment graph.offsets failed");
return IndexError_InvalidArgument;
}
const GraphNeighborMeta *neighbors_index = nullptr;
if (neighbors_meta->read(0, reinterpret_cast<const void **>(&neighbors_index),
neighbors_meta->data_size()) !=
neighbors_meta->data_size()) {
LOG_ERROR("Read segment %s data failed", kGraphOffsetsSegmentId.c_str());
return IndexError_InvalidArgument;
}
const char *neighbor_data;
for (node_id_t id = 0; id < doc_cnt(); ++id) {
size_t rd_size = neighbors_index[id].neighbor_cnt * sizeof(node_id_t);
if (ailego_unlikely(
neighbors_->read(neighbors_index[id].offset,
reinterpret_cast<const void **>(&neighbor_data),
rd_size) != rd_size)) {
LOG_ERROR("Read neighbors from segment failed");
return IndexError_ReadData;
}
// copy level 0 neighbors to fixed size neighbors memory
char *dst = fixed_neighbors_.get() + neighbors_size() * id;
*reinterpret_cast<uint32_t *>(dst) = neighbors_index[id].neighbor_cnt;
memcpy(dst + sizeof(uint32_t), neighbor_data, rd_size);
}
return 0;
}
int HnswRabitqSearcherEntity::get_fixed_neighbors(
std::vector<uint32_t> *fixed_neighbors) const {
//! Get a new segemnt to release the buffer after loading neighbors
auto neighbors_meta = storage_->get(kGraphOffsetsSegmentId);
if (!neighbors_meta) {
LOG_ERROR("IndexStorage get segment graph.offsets failed");
return IndexError_InvalidArgument;
}
const GraphNeighborMeta *neighbors_index = nullptr;
size_t meta_size = neighbors_meta->data_size();
if (neighbors_meta->read(0, reinterpret_cast<const void **>(&neighbors_index),
meta_size) != meta_size) {
LOG_ERROR("Read segment %s data failed", kGraphOffsetsSegmentId.c_str());
return IndexError_InvalidArgument;
}
size_t fixed_neighbor_cnt = l0_neighbor_cnt();
fixed_neighbors->resize((fixed_neighbor_cnt + 1) * doc_cnt(), kInvalidNodeId);
size_t neighbors_cnt_offset = fixed_neighbor_cnt * doc_cnt();
size_t total_neighbor_cnt = 0;
for (node_id_t id = 0; id < doc_cnt(); ++id) {
size_t cur_neighbor_cnt = neighbors_index[id].neighbor_cnt;
if (cur_neighbor_cnt == 0) {
(*fixed_neighbors)[neighbors_cnt_offset + id] = 0;
continue;
}
size_t rd_size = cur_neighbor_cnt * sizeof(node_id_t);
const uint32_t *neighbors;
if (neighbors_->read(neighbors_index[id].offset,
reinterpret_cast<const void **>(&neighbors),
rd_size) != rd_size) {
LOG_ERROR("Read neighbors from segment failed");
return IndexError_ReadData;
}
// copy level 0 neighbors to fixed size neighbors memory
auto it = fixed_neighbors->begin() + id * fixed_neighbor_cnt;
std::copy(neighbors, neighbors + cur_neighbor_cnt, it);
(*fixed_neighbors)[neighbors_cnt_offset + id] = cur_neighbor_cnt;
total_neighbor_cnt += cur_neighbor_cnt;
}
LOG_INFO("total neighbor cnt: %zu, average neighbor cnt: %zu",
total_neighbor_cnt, total_neighbor_cnt / doc_cnt());
return 0;
}
bool HnswRabitqSearcherEntity::do_crc_check(
std::vector<SegmentPointer> &segments) const {
constexpr size_t blk_size = 4096;
const void *data;
for (auto &segment : segments) {
size_t offset = 0;
size_t rd_size;
uint32_t crc = 0;
while (offset < segment->data_size()) {
size_t size = std::min(blk_size, segment->data_size() - offset);
if ((rd_size = segment->read(offset, &data, size)) <= 0) {
break;
}
offset += rd_size;
crc = ailego::Crc32c::Hash(data, rd_size, crc);
}
if (crc != segment->data_crc()) {
return false;
}
}
return true;
}
const HnswRabitqEntity::Pointer HnswRabitqSearcherEntity::clone() const {
auto vectors = vectors_->clone();
if (ailego_unlikely(!vectors)) {
LOG_ERROR("clone segment %s failed", kGraphFeaturesSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto keys = keys_->clone();
if (ailego_unlikely(!keys)) {
LOG_ERROR("clone segment %s failed", kGraphKeysSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto mapping = mapping_->clone();
if (ailego_unlikely(!mapping)) {
LOG_ERROR("clone segment %s failed", kGraphMappingSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto neighbors = neighbors_->clone();
if (ailego_unlikely(!neighbors)) {
LOG_ERROR("clone segment %s failed", kGraphNeighborsSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto upper_neighbors = upper_neighbors_->clone();
if (ailego_unlikely(!neighbors)) {
LOG_ERROR("clone segment %s failed", kHnswNeighborsSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto neighbors_meta = neighbors_meta_->clone();
if (ailego_unlikely(!neighbors_meta)) {
LOG_ERROR("clone segment %s failed", kGraphOffsetsSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
auto upper_neighbors_meta = upper_neighbors_meta_->clone();
if (ailego_unlikely(!upper_neighbors_meta)) {
LOG_ERROR("clone segment %s failed", kHnswOffsetsSegmentId.c_str());
return HnswRabitqEntity::Pointer();
}
SegmentGroupParam neighbor_group{neighbors, neighbors_meta, upper_neighbors,
upper_neighbors_meta};
HnswRabitqSearcherEntity *entity = new (std::nothrow)
HnswRabitqSearcherEntity(header(), vectors, keys, mapping, neighbor_group,
fixed_neighbors_, neighbors_in_memory_enabled_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswRabitqSearcherEntity new failed");
}
return HnswRabitqEntity::Pointer(entity);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,158 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "hnsw_rabitq_builder_entity.h"
#include "hnsw_rabitq_entity.h"
namespace zvec {
namespace core {
class HnswRabitqSearcherEntity : public HnswRabitqEntity {
public:
using Pointer = std::shared_ptr<HnswRabitqSearcherEntity>;
using SegmentPointer = IndexStorage::Segment::Pointer;
public:
struct SegmentGroupParam {
SegmentGroupParam(SegmentPointer neighbors_in,
SegmentPointer neighbors_meta_in,
SegmentPointer upper_neighbors_in,
SegmentPointer upper_neighbors_meta_in)
: neighbors{neighbors_in},
neighbors_meta{neighbors_meta_in},
upper_neighbors{upper_neighbors_in},
upper_neighbors_meta{upper_neighbors_meta_in} {}
SegmentPointer neighbors{nullptr};
SegmentPointer neighbors_meta{nullptr};
SegmentPointer upper_neighbors{nullptr};
SegmentPointer upper_neighbors_meta{nullptr};
};
//! Constructor
HnswRabitqSearcherEntity();
//! Make a copy of searcher entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswRabitqEntity::Pointer clone() const override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
//! Get vector local id by key
node_id_t get_id(key_t key) const;
//! Get vector feature data by key
virtual const void *get_vector_by_key(key_t key) const override;
//! Get vector feature data by id
virtual const void *get_vector(node_id_t id) const override;
//! Get vector feature data by id
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
//! Get all vectors
const void *get_vectors() const;
//! Get the node id's neighbors on graph level
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
virtual int load(const IndexStorage::Pointer &container,
bool check_crc) override;
int load_segments(bool check_crc);
virtual int cleanup(void) override;
public:
bool is_loaded() const {
return loaded_;
}
void set_neighbors_in_memory(bool enabled) {
neighbors_in_memory_enabled_ = enabled;
}
//! get fixed length neighbors data
int get_fixed_neighbors(std::vector<uint32_t> *fixed_neighbors) const;
private:
//! Constructor
HnswRabitqSearcherEntity(const HNSWHeader &hd, const SegmentPointer &vectors,
const SegmentPointer &keys,
const SegmentPointer &mapping,
const SegmentGroupParam &neighbor_group,
const std::shared_ptr<char> &fixed_neighbors,
bool neighbors_in_memory_enabled)
: HnswRabitqEntity(hd),
vectors_(vectors),
keys_(keys),
mapping_(mapping),
neighbors_(neighbor_group.neighbors),
neighbors_meta_(neighbor_group.neighbors_meta),
upper_neighbors_(neighbor_group.upper_neighbors),
upper_neighbors_meta_(neighbor_group.upper_neighbors_meta),
neighbors_in_memory_enabled_(neighbors_in_memory_enabled) {
segment_datas_.resize(std::max(l0_neighbor_cnt(), upper_neighbor_cnt()),
IndexStorage::SegmentData(0U, 0U));
fixed_neighbors_ = fixed_neighbors;
}
bool do_crc_check(std::vector<SegmentPointer> &segments) const;
inline size_t neighbors_size() const {
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
}
inline size_t upper_neighbors_size() const {
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
}
//! If neighbors_in_memory_enabled, load the level0 neighbors to memory
int load_and_flat_neighbors(void);
public:
HnswRabitqSearcherEntity(const HnswRabitqSearcherEntity &) = delete;
HnswRabitqSearcherEntity &operator=(const HnswRabitqSearcherEntity &) =
delete;
private:
IndexStorage::Pointer storage_{};
SegmentPointer vectors_{};
SegmentPointer keys_{};
SegmentPointer mapping_{};
SegmentPointer neighbors_{};
SegmentPointer neighbors_meta_{};
SegmentPointer upper_neighbors_{};
SegmentPointer upper_neighbors_meta_{};
mutable std::vector<IndexStorage::SegmentData> segment_datas_{};
std::shared_ptr<char> fixed_neighbors_{}; // level 0 fixed size neighbors
bool neighbors_in_memory_enabled_{false};
bool loaded_{false};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,989 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_streamer.h"
#include <iostream>
#include <memory>
#include <ailego/internal/cpu_features.h>
#include <ailego/pattern/defer.h>
#include <ailego/utility/memory_helper.h>
#include <arrow/compute/ordering.h>
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
#include "zvec/ailego/container/params.h"
#include "zvec/ailego/logger/logger.h"
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_context.h"
#include "hnsw_rabitq_dist_calculator.h"
#include "hnsw_rabitq_index_provider.h"
#include "hnsw_rabitq_query_entity.h"
#include "rabitq_params.h"
#include "rabitq_utils.h"
namespace zvec {
namespace core {
HnswRabitqStreamer::HnswRabitqStreamer() : entity_(stats_) {}
HnswRabitqStreamer::HnswRabitqStreamer(IndexProvider::Pointer provider,
RabitqReformer::Pointer reformer)
: entity_(stats_),
reformer_(std::move(reformer)),
provider_(std::move(provider)) {}
HnswRabitqStreamer::~HnswRabitqStreamer() {
if (state_ == STATE_INITED) {
this->cleanup();
}
}
int HnswRabitqStreamer::init(const IndexMeta &imeta,
const ailego::Params &params) {
meta_ = imeta;
meta_.set_streamer("HnswRabitqStreamer", HnswRabitqEntity::kRevision, params);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_INDEX_SIZE, &max_index_size_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT,
&upper_max_neighbor_cnt_);
float multiplier = HnswRabitqEntity::kDefaultL0MaxNeighborCntMultiplier;
params.get(PARAM_HNSW_RABITQ_STREAMER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER,
&multiplier);
l0_max_neighbor_cnt_ = multiplier * upper_max_neighbor_cnt_;
multiplier = HnswRabitqEntity::kDefaultNeighborPruneMultiplier;
params.get(PARAM_HNSW_RABITQ_STREAMER_NEIGHBOR_PRUNE_MULTIPLIER, &multiplier);
size_t prune_cnt = multiplier * upper_max_neighbor_cnt_;
scaling_factor_ = upper_max_neighbor_cnt_;
params.get(PARAM_HNSW_RABITQ_STREAMER_SCALING_FACTOR, &scaling_factor_);
params.get(PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT, &docs_hard_limit_);
params.get(PARAM_HNSW_RABITQ_STREAMER_EF, &ef_);
params.get(PARAM_HNSW_RABITQ_STREAMER_EFCONSTRUCTION, &ef_construction_);
params.get(PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_ENABLE, &bf_enabled_);
params.get(PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB,
&bf_negative_prob_);
params.get(PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
params.get(PARAM_HNSW_RABITQ_STREAMER_CHECK_CRC_ENABLE, &check_crc_enabled_);
params.get(PARAM_HNSW_RABITQ_STREAMER_CHUNK_SIZE, &chunk_size_);
params.get(PARAM_HNSW_RABITQ_STREAMER_FILTER_SAME_KEY, &filter_same_key_);
params.get(PARAM_HNSW_RABITQ_STREAMER_GET_VECTOR_ENABLE,
&get_vector_enabled_);
params.get(PARAM_HNSW_RABITQ_STREAMER_MIN_NEIGHBOR_COUNT, &min_neighbor_cnt_);
params.get(PARAM_HNSW_RABITQ_STREAMER_FORCE_PADDING_RESULT_ENABLE,
&force_padding_topk_enabled_);
params.get(PARAM_HNSW_RABITQ_STREAMER_USE_ID_MAP, &use_id_map_);
entity_.set_use_key_info_map(use_id_map_);
params.get(PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT, &docs_soft_limit_);
if (docs_soft_limit_ > 0 && docs_soft_limit_ > docs_hard_limit_) {
LOG_ERROR("[%s] must be >= [%s]",
PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT.c_str(),
PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT.c_str());
return IndexError_InvalidArgument;
} else if (docs_soft_limit_ == 0UL) {
docs_soft_limit_ =
docs_hard_limit_ * HnswRabitqEntity::kDefaultDocsSoftLimitRatio;
}
if (ef_ == 0U) {
ef_ = HnswRabitqEntity::kDefaultEf;
}
if (ef_construction_ == 0U) {
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
}
if (upper_max_neighbor_cnt_ == 0U) {
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
}
if (upper_max_neighbor_cnt_ > HnswRabitqEntity::kMaxNeighborCnt) {
LOG_ERROR("[%s] must be in range (0,%d)",
PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT.c_str(),
HnswRabitqEntity::kMaxNeighborCnt);
return IndexError_InvalidArgument;
}
if (l0_max_neighbor_cnt_ == 0U) {
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultL0MaxNeighborCnt;
}
if (l0_max_neighbor_cnt_ > HnswRabitqEntity::kMaxNeighborCnt) {
LOG_ERROR("MaxL0NeighborCnt must be in range (0,%d)",
HnswRabitqEntity::kMaxNeighborCnt);
return IndexError_InvalidArgument;
}
if (min_neighbor_cnt_ > upper_max_neighbor_cnt_) {
LOG_ERROR("[%s]-[%zu] must be <= [%s]-[%zu]",
PARAM_HNSW_RABITQ_STREAMER_MIN_NEIGHBOR_COUNT.c_str(),
static_cast<size_t>(min_neighbor_cnt_),
PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT.c_str(),
static_cast<size_t>(upper_max_neighbor_cnt_));
return IndexError_InvalidArgument;
}
if (bf_negative_prob_ <= 0.0f || bf_negative_prob_ >= 1.0f) {
LOG_ERROR(
"[%s] must be in range (0,1)",
PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB.c_str());
return IndexError_InvalidArgument;
}
if (scaling_factor_ == 0U) {
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
}
if (scaling_factor_ < 5 || scaling_factor_ > 1000) {
LOG_ERROR("[%s] must be in range [5,1000]",
PARAM_HNSW_RABITQ_STREAMER_SCALING_FACTOR.c_str());
return IndexError_InvalidArgument;
}
if (max_scan_ratio_ <= 0.0f || max_scan_ratio_ > 1.0f) {
LOG_ERROR("[%s] must be in range (0.0f,1.0f]",
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO.c_str());
return IndexError_InvalidArgument;
}
if (max_scan_limit_ < min_scan_limit_) {
LOG_ERROR("[%s] must be >= [%s]",
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT.c_str(),
PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT.c_str());
return IndexError_InvalidArgument;
}
if (prune_cnt == 0UL) {
prune_cnt = upper_max_neighbor_cnt_;
}
if (chunk_size_ == 0UL) {
chunk_size_ = HnswRabitqEntity::kDefaultChunkSize;
}
if (chunk_size_ > HnswRabitqEntity::kMaxChunkSize) {
LOG_ERROR("[%s] must be < %zu",
PARAM_HNSW_RABITQ_STREAMER_CHUNK_SIZE.c_str(),
HnswRabitqEntity::kMaxChunkSize);
return IndexError_InvalidArgument;
}
uint32_t total_bits = 0;
params.get(PARAM_RABITQ_TOTAL_BITS, &total_bits);
if (total_bits == 0) {
total_bits = kDefaultRabitqTotalBits;
}
if (total_bits < 1 || total_bits > 9) {
LOG_ERROR("Invalid total_bits: %zu, must be in [1, 9]", (size_t)total_bits);
return IndexError_InvalidArgument;
}
uint8_t ex_bits = total_bits - 1;
entity_.set_ex_bits(ex_bits);
uint32_t dimension = 0;
params.get(PARAM_HNSW_RABITQ_GENERAL_DIMENSION, &dimension);
if (dimension == 0) {
LOG_ERROR("%s not set", PARAM_HNSW_RABITQ_GENERAL_DIMENSION.c_str());
return IndexError_InvalidArgument;
}
if (dimension < kMinRabitqDimSize || dimension > kMaxRabitqDimSize) {
LOG_ERROR("Invalid dimension: %u, must be in [%d, %d]", dimension,
kMinRabitqDimSize, kMaxRabitqDimSize);
return IndexError_InvalidArgument;
}
entity_.update_rabitq_params_and_vector_size(dimension);
entity_.set_ef_construction(ef_construction_);
entity_.set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
entity_.set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
entity_.set_scaling_factor(scaling_factor_);
entity_.set_prune_cnt(prune_cnt);
entity_.set_chunk_size(chunk_size_);
entity_.set_filter_same_key(filter_same_key_);
entity_.set_get_vector(get_vector_enabled_);
entity_.set_min_neighbor_cnt(min_neighbor_cnt_);
int ret = entity_.init(docs_hard_limit_);
if (ret != 0) {
LOG_ERROR("Hnsw entity init failed for %s", IndexError::What(ret));
return ret;
}
LOG_DEBUG(
"Init params: maxIndexSize=%zu docsHardLimit=%zu docsSoftLimit=%zu "
"efConstruction=%u ef=%u upperMaxNeighborCnt=%u l0MaxNeighborCnt=%u "
"scalingFactor=%u maxScanRatio=%.3f minScanLimit=%zu maxScanLimit=%zu "
"bfEnabled=%d bruteFoceThreshold=%zu bfNegativeProbability=%.5f "
"checkCrcEnabled=%d pruneSize=%zu vectorSize=%u chunkSize=%zu "
"filterSameKey=%u getVectorEnabled=%u minNeighborCount=%u "
"forcePadding=%u ",
max_index_size_, docs_hard_limit_, docs_soft_limit_, ef_construction_,
ef_, upper_max_neighbor_cnt_, l0_max_neighbor_cnt_, scaling_factor_,
max_scan_ratio_, min_scan_limit_, max_scan_limit_, bf_enabled_,
bruteforce_threshold_, bf_negative_prob_, check_crc_enabled_, prune_cnt,
meta_.element_size(), chunk_size_, filter_same_key_, get_vector_enabled_,
min_neighbor_cnt_, force_padding_topk_enabled_);
alg_ = HnswRabitqAlgorithm::UPointer(new HnswRabitqAlgorithm(entity_));
ret = alg_->init();
if (ret != 0) {
return ret;
}
state_ = STATE_INITED;
return 0;
}
int HnswRabitqStreamer::cleanup(void) {
if (state_ == STATE_OPENED) {
this->close();
}
LOG_INFO("HnswRabitqStreamer cleanup");
meta_.clear();
metric_.reset();
stats_.clear();
entity_.cleanup();
if (alg_) {
alg_->cleanup();
}
max_index_size_ = 0UL;
docs_hard_limit_ = HnswRabitqEntity::kDefaultDocsHardLimit;
docs_soft_limit_ = 0UL;
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultL0MaxNeighborCnt;
ef_ = HnswRabitqEntity::kDefaultEf;
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
bf_enabled_ = false;
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
bruteforce_threshold_ = HnswRabitqEntity::kDefaultBruteForceThreshold;
max_scan_limit_ = HnswRabitqEntity::kDefaultMaxScanLimit;
min_scan_limit_ = HnswRabitqEntity::kDefaultMinScanLimit;
chunk_size_ = HnswRabitqEntity::kDefaultChunkSize;
bf_negative_prob_ = HnswRabitqEntity::kDefaultBFNegativeProbability;
max_scan_ratio_ = HnswRabitqEntity::kDefaultScanRatio;
state_ = STATE_INIT;
check_crc_enabled_ = false;
filter_same_key_ = false;
get_vector_enabled_ = false;
return 0;
}
int HnswRabitqStreamer::open(IndexStorage::Pointer stg) {
LOG_INFO("HnswRabitqStreamer open");
if (ailego_unlikely(state_ != STATE_INITED)) {
LOG_ERROR("Open storage failed, init streamer first!");
return IndexError_NoReady;
}
// try to load reformer
if (reformer_ == nullptr) {
reformer_ = std::make_shared<RabitqReformer>();
ailego::Params reformer_params;
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
int ret = reformer_->init(reformer_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
return ret;
}
ret = reformer_->load(stg);
if (ret != 0) {
LOG_ERROR("Failed to load reformer, ret=%d", ret);
return ret;
}
} else {
if (!stg->has(RABITQ_CONVERTER_SEG_ID)) {
int ret = reformer_->dump(stg);
if (ret != 0) {
LOG_ERROR("Failed to dump reformer, ret=%d", ret);
return ret;
}
LOG_INFO("Dump reformer success.");
}
}
int ret = entity_.open(std::move(stg), max_index_size_, check_crc_enabled_);
if (ret != 0) {
return ret;
}
IndexMeta index_meta;
ret = entity_.get_index_meta(&index_meta);
if (ret == IndexError_NoExist) {
// Set IndexMeta for the new index
ret = entity_.set_index_meta(meta_);
if (ret != 0) {
LOG_ERROR("Failed to set index meta for %s", IndexError::What(ret));
return ret;
}
} else if (ret != 0) {
LOG_ERROR("Failed to get index meta for %s", IndexError::What(ret));
return ret;
} else {
if (index_meta.dimension() != meta_.dimension() ||
index_meta.element_size() != meta_.element_size() ||
index_meta.metric_name() != meta_.metric_name() ||
index_meta.data_type() != meta_.data_type()) {
LOG_ERROR("IndexMeta mismatch from the previous in index");
return IndexError_Mismatch;
}
// The IndexMetric Params may be updated like MipsSquaredEuclidean
auto metric_params = index_meta.metric_params();
metric_params.merge(meta_.metric_params());
meta_.set_metric(index_meta.metric_name(), 0, metric_params);
}
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!metric_) {
LOG_ERROR("Failed to create metric %s", meta_.metric_name().c_str());
return IndexError_NoExist;
}
ret = metric_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("Failed to init metric, ret=%d", ret);
return ret;
}
if (!metric_->distance()) {
LOG_ERROR("Invalid metric distance");
return IndexError_InvalidArgument;
}
if (!metric_->batch_distance()) {
LOG_ERROR("Invalid metric batch distance");
return IndexError_InvalidArgument;
}
add_distance_ = metric_->distance();
add_batch_distance_ = metric_->batch_distance();
search_distance_ = add_distance_;
search_batch_distance_ = add_batch_distance_;
if (metric_->query_metric() && metric_->query_metric()->distance() &&
metric_->query_metric()->batch_distance()) {
search_distance_ = metric_->query_metric()->distance();
search_batch_distance_ = metric_->query_metric()->batch_distance();
}
state_ = STATE_OPENED;
magic_ = IndexContext::GenerateMagic();
query_alg_ = HnswRabitqQueryAlgorithm::UPointer(new HnswRabitqQueryAlgorithm(
entity_, reformer_->num_clusters(), reformer_->rabitq_metric_type()));
return 0;
}
int HnswRabitqStreamer::close(void) {
LOG_INFO("HnswRabitqStreamer close");
stats_.clear();
meta_.set_metric(metric_->name(), 0, metric_->params());
entity_.set_index_meta(meta_);
int ret = entity_.close();
if (ret != 0) {
return ret;
}
state_ = STATE_INITED;
return 0;
}
int HnswRabitqStreamer::flush(uint64_t checkpoint) {
LOG_INFO("HnswRabitqStreamer flush checkpoint=%zu", (size_t)checkpoint);
meta_.set_metric(metric_->name(), 0, metric_->params());
entity_.set_index_meta(meta_);
return entity_.flush(checkpoint);
}
int HnswRabitqStreamer::dump(const IndexDumper::Pointer &dumper) {
LOG_INFO("HnswRabitqStreamer dump");
shared_mutex_.lock();
AILEGO_DEFER([&]() { shared_mutex_.unlock(); });
meta_.set_searcher("HnswRabitqSearcher", HnswRabitqEntity::kRevision,
ailego::Params());
int ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
if (ret != 0) {
LOG_ERROR("Failed to serialize meta into dumper.");
return ret;
}
ret = reformer_->dump(dumper);
if (ret != 0) {
LOG_ERROR("Failed to dump reformer into dumper.");
return ret;
}
return entity_.dump(dumper);
}
IndexStreamer::Context::Pointer HnswRabitqStreamer::create_context(void) const {
if (ailego_unlikely(state_ != STATE_OPENED)) {
LOG_ERROR("Create context failed, open storage first!");
return Context::Pointer();
}
HnswRabitqEntity::Pointer entity = entity_.clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("CreateContext clone init failed");
return Context::Pointer();
}
HnswRabitqContext *ctx =
new (std::nothrow) HnswRabitqContext(meta_.dimension(), metric_, entity);
if (ailego_unlikely(ctx == nullptr)) {
LOG_ERROR("Failed to new HnswRabitqContext");
return Context::Pointer();
}
ctx->set_ef(ef_);
ctx->set_max_scan_limit(max_scan_limit_);
ctx->set_min_scan_limit(min_scan_limit_);
ctx->set_max_scan_ratio(max_scan_ratio_);
ctx->set_filter_mode(bf_enabled_ ? VisitFilter::BloomFilter
: VisitFilter::ByteMap);
ctx->set_filter_negative_probability(bf_negative_prob_);
ctx->set_magic(magic_);
ctx->set_force_padding_topk(force_padding_topk_enabled_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
if (ailego_unlikely(ctx->init(HnswRabitqContext::kStreamerContext)) != 0) {
LOG_ERROR("Init HnswRabitqContext failed");
delete ctx;
return Context::Pointer();
}
uint32_t estimate_doc_count = 0;
if (meta_.streamer_params().get(PARAM_HNSW_RABITQ_STREAMER_ESTIMATE_DOC_COUNT,
&estimate_doc_count)) {
LOG_DEBUG("HnswRabitqStreamer doc_count[%zu] estimate[%zu]",
(size_t)entity_.doc_cnt(), (size_t)estimate_doc_count);
}
ctx->check_need_adjuct_ctx(std::max(entity_.doc_cnt(), estimate_doc_count));
return Context::Pointer(ctx);
}
IndexProvider::Pointer HnswRabitqStreamer::create_provider(void) const {
LOG_DEBUG("HnswRabitqStreamer create provider");
auto entity = entity_.clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("Clone HnswRabitqEntity failed");
return nullptr;
}
return Provider::Pointer(
new HnswRabitqIndexProvider(meta_, entity, "HnswRabitqStreamer"));
}
int HnswRabitqStreamer::update_context(HnswRabitqContext *ctx) const {
const HnswRabitqEntity::Pointer entity = entity_.clone();
if (!entity) {
LOG_ERROR("Failed to clone search context entity");
return IndexError_Runtime;
}
ctx->set_max_scan_limit(max_scan_limit_);
ctx->set_min_scan_limit(min_scan_limit_);
ctx->set_max_scan_ratio(max_scan_ratio_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
return ctx->update_context(HnswRabitqContext::kStreamerContext, meta_,
metric_, entity, magic_);
}
//! Add a vector with id into index
int HnswRabitqStreamer::add_with_id_impl(
uint32_t id, const void *query, const IndexQueryMeta &qmeta,
IndexStreamer::Context::Pointer &context) {
if (!provider_) {
LOG_ERROR("Provider is nullptr, cannot add vector");
return IndexError_InvalidArgument;
}
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
if (entity_.doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %zu exceed [%s]",
static_cast<size_t>(entity_.doc_cnt()),
PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT.c_str());
const std::lock_guard<std::mutex> lk(mutex_);
(*stats_.mutable_discarded_count())++;
return IndexError_IndexFull;
} else {
LOG_WARN("Current docs %zu exceed [%s]",
static_cast<size_t>(entity_.doc_cnt()),
PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT.c_str());
}
}
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
LOG_ERROR("Cannot add vector while dumping index");
(*stats_.mutable_discarded_count())++;
return IndexError_Unsupported;
}
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
ctx->set_provider(provider_);
if (metric_->support_train()) {
const std::lock_guard<std::mutex> lk(mutex_);
ret = metric_->train(query, meta_.dimension());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer metric train failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
}
std::string converted_vector;
IndexQueryMeta converted_meta;
ret = reformer_->convert(query, qmeta, &converted_vector, &converted_meta);
if (ret != 0) {
LOG_ERROR("Rabitq hnsw convert failed, ret=%d", ret);
return ret;
}
level_t level = alg_->get_random_level();
ret = entity_.add_vector_with_id(level, id, converted_vector.data());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer add vector failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
ret = alg_->add_node(id, level, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw steamer add node failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
if (ailego_unlikely(ctx->error())) {
(*stats_.mutable_discarded_count())++;
return IndexError_Runtime;
}
(*stats_.mutable_added_count())++;
return 0;
}
//! Add a vector into index
int HnswRabitqStreamer::add_impl(uint64_t pkey, const void *query,
const IndexQueryMeta &qmeta,
IndexStreamer::Context::Pointer &context) {
if (!provider_) {
LOG_ERROR("Provider is nullptr, cannot add vector");
return IndexError_InvalidArgument;
}
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
if (entity_.doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %zu exceed [%s]",
static_cast<size_t>(entity_.doc_cnt()),
PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT.c_str());
const std::lock_guard<std::mutex> lk(mutex_);
(*stats_.mutable_discarded_count())++;
return IndexError_IndexFull;
} else {
LOG_WARN("Current docs %zu exceed [%s]",
static_cast<size_t>(entity_.doc_cnt()),
PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT.c_str());
}
}
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
LOG_ERROR("Cannot add vector while dumping index");
(*stats_.mutable_discarded_count())++;
return IndexError_Unsupported;
}
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
ctx->set_provider(provider_);
if (metric_->support_train()) {
const std::lock_guard<std::mutex> lk(mutex_);
ret = metric_->train(query, meta_.dimension());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer metric train failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
}
std::string converted_vector;
IndexQueryMeta converted_meta;
ret = reformer_->convert(query, qmeta, &converted_vector, &converted_meta);
if (ret != 0) {
LOG_ERROR("Rabitq hnsw convert failed, ret=%d", ret);
return ret;
}
level_t level = alg_->get_random_level();
node_id_t id;
ret = entity_.add_vector(level, pkey, converted_vector.data(), &id);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer add vector failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
ret = alg_->add_node(id, level, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw steamer add node failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
if (ailego_unlikely(ctx->error())) {
(*stats_.mutable_discarded_count())++;
return IndexError_Runtime;
}
(*stats_.mutable_added_count())++;
return 0;
}
int HnswRabitqStreamer::search_impl(
const void *query, const IndexQueryMeta &qmeta,
IndexStreamer::Context::Pointer &context) const {
return search_impl(query, qmeta, 1, context);
}
//! Similarity search
int HnswRabitqStreamer::search_impl(
const void *query, const IndexQueryMeta &qmeta, uint32_t count,
IndexStreamer::Context::Pointer &context) const {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (entity_.doc_cnt() <= ctx->get_bruteforce_threshold()) {
return search_bf_impl(query, qmeta, count, context);
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher transform failed");
return ret;
}
ctx->reset_query(query);
ret = query_alg_->search(&entity, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw searcher fast search failed");
return ret;
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
void HnswRabitqStreamer::print_debug_info() {
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
Neighbors neighbours = entity_.get_neighbors(0, id);
std::cout << "node: " << id << "; ";
if (neighbours.size() == 0) std::cout << std::endl;
for (uint32_t i = 0; i < neighbours.size(); ++i) {
std::cout << neighbours[i];
if (i == neighbours.size() - 1) {
std::cout << std::endl;
} else {
std::cout << ", ";
}
}
}
// entity_.print_key_map();
}
int HnswRabitqStreamer::search_bf_impl(
const void *query, const IndexQueryMeta &qmeta,
IndexStreamer::Context::Pointer &context) const {
return search_bf_impl(query, qmeta, 1, context);
}
int HnswRabitqStreamer::search_bf_impl(
const void *query, const IndexQueryMeta &qmeta, uint32_t count,
IndexStreamer::Context::Pointer &context) const {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
if (ctx->group_by_search()) {
if (!ctx->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
};
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw rabitq streamer transform failed");
return ret;
}
ctx->reset_query(query);
ctx->group_topk_heaps().clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
EstimateRecord dist;
query_alg_->get_full_est(id, dist, entity);
std::string group_id = group_by(id);
auto &topk_heap = ctx->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
} else {
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw rabitq streamer transform failed");
return ret;
}
ctx->reset_query(query);
ctx->topk_heap().clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
continue;
}
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
EstimateRecord dist;
query_alg_->get_full_est(id, dist, entity);
ctx->topk_heap().emplace(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
int HnswRabitqStreamer::search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) {
return ret;
}
if (ailego_unlikely(p_keys.size() != count)) {
LOG_ERROR("The size of p_keys is not equal to count");
return IndexError_InvalidArgument;
}
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to HnswRabitqContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
//! context is created by another searcher or streamer
ret = update_context(ctx);
if (ret != 0) {
return ret;
}
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
if (ctx->group_by_search()) {
if (!ctx->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
};
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw rabitq streamer transform failed");
return ret;
}
ctx->reset_query(query);
ctx->group_topk_heaps().clear();
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
uint64_t pk = p_keys[q][idx];
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
node_id_t id = entity_.get_id(pk);
if (id != kInvalidNodeId) {
EstimateRecord dist;
query_alg_->get_full_est(id, dist, entity);
std::string group_id = group_by(id);
auto &topk_heap = ctx->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(ctx->group_topk());
}
topk_heap.emplace_back(id, dist);
}
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
} else {
for (size_t q = 0; q < count; ++q) {
HnswRabitqQueryEntity entity;
ret = reformer_->transform_to_entity(query, &entity);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw rabitq streamer transform failed");
return ret;
}
ctx->reset_query(query);
ctx->topk_heap().clear();
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
key_t pk = p_keys[q][idx];
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
node_id_t id = entity_.get_id(pk);
if (id != kInvalidNodeId) {
EstimateRecord dist;
query_alg_->get_full_est(id, dist, entity);
ctx->topk_heap().emplace(id, dist);
}
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
}
if (ailego_unlikely(ctx->error())) {
return IndexError_Runtime;
}
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,249 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <ailego/parallel/lock.h>
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
#include "zvec/core/framework/index_framework.h"
#include "zvec/core/framework/index_provider.h"
#include "zvec/core/framework/index_reformer.h"
#include "hnsw_rabitq_algorithm.h"
#include "hnsw_rabitq_query_algorithm.h"
#include "hnsw_rabitq_streamer_entity.h"
namespace zvec {
namespace core {
class HnswRabitqStreamer : public IndexStreamer {
public:
using ContextPointer = IndexStreamer::Context::Pointer;
HnswRabitqStreamer();
explicit HnswRabitqStreamer(IndexProvider::Pointer provider,
RabitqReformer::Pointer reformer = nullptr);
virtual ~HnswRabitqStreamer(void);
HnswRabitqStreamer(const HnswRabitqStreamer &streamer) = delete;
HnswRabitqStreamer &operator=(const HnswRabitqStreamer &streamer) = delete;
void set_provider(IndexProvider::Pointer provider) {
provider_ = std::move(provider);
}
void set_reformer(IndexReformer::Pointer reformer) {
reformer_ = std::dynamic_pointer_cast<RabitqReformer>(reformer);
}
protected:
//! Initialize Streamer
virtual int init(const IndexMeta &imeta,
const ailego::Params &params) override;
//! Cleanup Streamer
virtual int cleanup(void) override;
//! Create a context
virtual Context::Pointer create_context(void) const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
//! Add a vector into index
virtual int add_impl(uint64_t pkey, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Add a vector with id into index
virtual int add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
//! Fetch vector by key
virtual const void *get_vector(uint64_t key) const override {
return entity_.get_vector_by_key(key);
}
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_.get_vector_by_key(key, block);
}
//! Fetch vector by id
virtual const void *get_vector_by_id(uint32_t id) const override {
return entity_.get_vector(id);
}
virtual int get_vector_by_id(
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
return entity_.get_vector(id, block);
}
//! Open index from file path
virtual int open(IndexStorage::Pointer stg) override;
//! Close file
virtual int close(void) override;
//! flush file
virtual int flush(uint64_t checkpoint) override;
//! Dump index into storage
virtual int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
return meta_;
}
virtual void print_debug_info() override;
private:
inline int check_params(const void *query,
const IndexQueryMeta &qmeta) const {
if (ailego_unlikely(!query)) {
LOG_ERROR("null query");
return IndexError_InvalidArgument;
}
if (ailego_unlikely(qmeta.dimension() != meta_.dimension() ||
qmeta.data_type() != meta_.data_type() ||
qmeta.element_size() != meta_.element_size())) {
LOG_ERROR("Unsupported query meta");
return IndexError_Mismatch;
}
return 0;
}
inline int check_sparse_count_is_zero(const uint32_t *sparse_count,
uint32_t count) const {
for (uint32_t i = 0; i < count; ++i) {
if (sparse_count[i] != 0)
LOG_ERROR("Sparse cout is not empty. Index: %u, Sparse Count: %u", i,
sparse_count[i]);
return IndexError_InvalidArgument;
}
return 0;
}
private:
//! To share ctx across streamer/searcher, we need to update the context for
//! current streamer/searcher
int update_context(HnswRabitqContext *ctx) const;
private:
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_OPENED = 2 };
class Stats : public IndexStreamer::Stats {
public:
void clear(void) {
set_revision_id(0u);
set_loaded_count(0u);
set_added_count(0u);
set_discarded_count(0u);
set_index_size(0u);
set_dumped_size(0u);
set_check_point(0u);
set_create_time(0u);
set_update_time(0u);
clear_attributes();
}
};
HnswRabitqStreamerEntity entity_;
HnswRabitqAlgorithm::UPointer alg_;
IndexMeta meta_{};
IndexMetric::Pointer metric_{};
IndexMetric::MatrixDistance add_distance_{};
IndexMetric::MatrixDistance search_distance_{};
IndexMetric::MatrixBatchDistance add_batch_distance_{};
IndexMetric::MatrixBatchDistance search_batch_distance_{};
RabitqReformer::Pointer reformer_{}; // RaBitQ reformer
HnswRabitqQueryAlgorithm::UPointer query_alg_; // query algorithm
// provider_ provides raw vector, which is used to build graph
IndexProvider::Pointer provider_{};
Stats stats_{};
std::mutex mutex_{};
size_t max_index_size_{0UL};
size_t chunk_size_{HnswRabitqEntity::kDefaultChunkSize};
size_t docs_hard_limit_{HnswRabitqEntity::kDefaultDocsHardLimit};
size_t docs_soft_limit_{0UL};
uint32_t min_neighbor_cnt_{0u};
uint32_t upper_max_neighbor_cnt_{
HnswRabitqEntity::kDefaultUpperMaxNeighborCnt};
uint32_t l0_max_neighbor_cnt_{HnswRabitqEntity::kDefaultL0MaxNeighborCnt};
uint32_t ef_{HnswRabitqEntity::kDefaultEf};
uint32_t ef_construction_{HnswRabitqEntity::kDefaultEfConstruction};
uint32_t scaling_factor_{HnswRabitqEntity::kDefaultScalingFactor};
size_t bruteforce_threshold_{HnswRabitqEntity::kDefaultBruteForceThreshold};
size_t max_scan_limit_{HnswRabitqEntity::kDefaultMaxScanLimit};
size_t min_scan_limit_{HnswRabitqEntity::kDefaultMinScanLimit};
float bf_negative_prob_{HnswRabitqEntity::kDefaultBFNegativeProbability};
float max_scan_ratio_{HnswRabitqEntity::kDefaultScanRatio};
uint32_t magic_{0U};
State state_{STATE_INIT};
bool bf_enabled_{false};
bool check_crc_enabled_{false};
bool filter_same_key_{false};
bool get_vector_enabled_{false};
bool force_padding_topk_enabled_{false};
bool use_id_map_{true};
//! avoid add vector while dumping index
ailego::SharedMutex shared_mutex_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,709 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_rabitq_streamer_entity.h"
#include <ailego/utility/memory_helper.h>
// #define DEBUG_PRINT
namespace zvec {
namespace core {
HnswRabitqStreamerEntity::HnswRabitqStreamerEntity(IndexStreamer::Stats &stats)
: stats_(stats) {}
HnswRabitqStreamerEntity::~HnswRabitqStreamerEntity() {}
int HnswRabitqStreamerEntity::init(size_t max_doc_cnt) {
if (std::pow(scaling_factor(), kMaxGraphLayers) < max_doc_cnt) {
LOG_ERROR("scalingFactor=%zu is too small", scaling_factor());
return IndexError_InvalidArgument;
}
std::lock_guard<std::mutex> lock(mutex_);
broker_ = std::make_shared<HnswRabitqChunkBroker>(stats_);
upper_neighbor_index_ = std::make_shared<NIHashMap>();
keys_map_lock_ = std::make_shared<ailego::SharedMutex>();
keys_map_ = std::make_shared<HashMap<key_t, node_id_t>>();
if (!keys_map_ || !upper_neighbor_index_ || !broker_ || !keys_map_lock_) {
LOG_ERROR("HnswRabitqStreamerEntity new object failed");
return IndexError_NoMemory;
}
keys_map_->set_empty_key(kInvalidKey);
neighbor_size_ = neighbors_size();
upper_neighbor_size_ = upper_neighbors_size();
//! vector + key + level 0 neighbors
size_t size = vector_size() + sizeof(key_t) + neighbor_size_;
size = AlignSize(size);
set_node_size(size);
return 0;
}
int HnswRabitqStreamerEntity::cleanup() {
std::lock_guard<std::mutex> lock(mutex_);
mutable_header()->clear();
chunk_size_ = kDefaultChunkSize;
node_index_mask_bits_ = 0U;
node_index_mask_ = 0U;
node_cnt_per_chunk_ = 0U;
neighbor_size_ = 0U;
upper_neighbor_size_ = 0U;
if (upper_neighbor_index_) {
upper_neighbor_index_->cleanup();
}
if (keys_map_) {
keys_map_->clear();
}
node_chunks_.clear();
upper_neighbor_chunks_.clear();
filter_same_key_ = false;
get_vector_enabled_ = false;
broker_.reset();
return 0;
}
int HnswRabitqStreamerEntity::update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors) {
char buffer[neighbor_size_];
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer);
hd->neighbor_cnt = neighbors.size();
size_t i = 0;
for (; i < neighbors.size(); ++i) {
hd->neighbors[i] = neighbors[i].first;
}
auto loc = get_neighbor_chunk_loc(level, id);
size_t size = reinterpret_cast<char *>(&hd->neighbors[i]) - &buffer[0];
size_t ret = loc.first->write(loc.second, hd, size);
if (ailego_unlikely(ret != size)) {
LOG_ERROR("Write neighbor header failed, ret=%zu", ret);
return IndexError_Runtime;
}
return 0;
}
const Neighbors HnswRabitqStreamerEntity::get_neighbors(level_t level,
node_id_t id) const {
Chunk *chunk = nullptr;
size_t offset = 0UL;
size_t neighbor_size = neighbor_size_;
if (level == 0UL) {
uint32_t chunk_idx = id >> node_index_mask_bits_;
offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
&node_chunks_);
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
chunk = node_chunks_[chunk_idx].get();
} else {
auto p = get_upper_neighbor_chunk_loc(level, id);
chunk = upper_neighbor_chunks_[p.first].get();
offset = p.second;
neighbor_size = upper_neighbor_size_;
}
ailego_assert_with(offset < chunk->data_size(), "invalid chunk offset");
IndexStorage::MemoryBlock neighbor_block;
size_t size = chunk->read(offset, neighbor_block, neighbor_size);
if (ailego_unlikely(size != neighbor_size)) {
LOG_ERROR("Read neighbor header failed, ret=%zu", size);
return Neighbors();
}
return Neighbors(std::move(neighbor_block));
}
//! Get vector data by key
const void *HnswRabitqStreamerEntity::get_vector(node_id_t id) const {
auto loc = get_vector_chunk_loc(id);
const void *vec = nullptr;
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
size_t ret = node_chunks_[loc.first]->read(loc.second, &vec, read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
static_cast<size_t>(loc.second), read_size, ret);
}
return vec;
}
int HnswRabitqStreamerEntity::get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const {
for (auto i = 0U; i < count; ++i) {
auto loc = get_vector_chunk_loc(ids[i]);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
size_t ret = node_chunks_[loc.first]->read(loc.second, &vecs[i], read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
static_cast<size_t>(loc.second), read_size, ret);
return IndexError_ReadData;
}
}
return 0;
}
int HnswRabitqStreamerEntity::get_vector(
const node_id_t id, IndexStorage::MemoryBlock &block) const {
auto loc = get_vector_chunk_loc(id);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
size_t ret = node_chunks_[loc.first]->read(loc.second, block, read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
static_cast<size_t>(loc.second), read_size, ret);
return IndexError_ReadData;
}
return 0;
}
int HnswRabitqStreamerEntity::get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
auto loc = get_vector_chunk_loc(ids[i]);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
size_t ret =
node_chunks_[loc.first]->read(loc.second, vec_blocks[i], read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
static_cast<size_t>(loc.second), read_size, ret);
return IndexError_ReadData;
}
}
return 0;
}
key_t HnswRabitqStreamerEntity::get_key(node_id_t id) const {
if (use_key_info_map_) {
auto loc = get_key_chunk_loc(id);
IndexStorage::MemoryBlock key_block;
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t ret =
node_chunks_[loc.first]->read(loc.second, key_block, sizeof(key_t));
if (ailego_unlikely(ret != sizeof(key_t))) {
LOG_ERROR("Read vector failed, ret=%zu", ret);
return kInvalidKey;
}
return *reinterpret_cast<const key_t *>(key_block.data());
} else {
return id;
}
}
void HnswRabitqStreamerEntity::add_neighbor(level_t level, node_id_t id,
uint32_t size,
node_id_t neighbor_id) {
auto loc = get_neighbor_chunk_loc(level, id);
size_t offset =
loc.second + sizeof(NeighborsHeader) + size * sizeof(node_id_t);
ailego_assert_with(size < neighbor_cnt(level), "invalid neighbor size");
ailego_assert_with(offset < loc.first->data_size(), "invalid chunk offset");
size_t ret = loc.first->write(offset, &neighbor_id, sizeof(node_id_t));
if (ailego_unlikely(ret != sizeof(node_id_t))) {
LOG_ERROR("Write neighbor id failed, ret=%zu", ret);
return;
}
uint32_t neighbors = size + 1;
ret = loc.first->write(loc.second, &neighbors, sizeof(uint32_t));
if (ailego_unlikely(ret != sizeof(uint32_t))) {
LOG_ERROR("Write neighbor cnt failed, ret=%zu", ret);
}
return;
}
int HnswRabitqStreamerEntity::init_chunks(const Chunk::Pointer &header_chunk) {
if (header_chunk->data_size() < header_size()) {
LOG_ERROR("Invalid header chunk size");
return IndexError_InvalidFormat;
}
IndexStorage::MemoryBlock header_block;
size_t size = header_chunk->read(0UL, header_block, header_size());
if (ailego_unlikely(size != header_size())) {
LOG_ERROR("Read header chunk failed");
return IndexError_ReadData;
}
*mutable_header() =
*reinterpret_cast<const HNSWHeader *>(header_block.data());
int ret = check_hnsw_index(&header());
if (ret != 0) {
broker_->close();
return ret;
}
node_chunks_.resize(
broker_->get_chunk_cnt(HnswRabitqChunkBroker::CHUNK_TYPE_NODE));
for (auto seq = 0UL; seq < node_chunks_.size(); ++seq) {
node_chunks_[seq] =
broker_->get_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, seq);
if (!node_chunks_[seq]) {
LOG_ERROR("Missing hnsw streamer data chunk %zu th of %zu", seq,
node_chunks_.size());
return IndexError_InvalidFormat;
}
}
upper_neighbor_chunks_.resize(
broker_->get_chunk_cnt(HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR));
for (auto seq = 0UL; seq < upper_neighbor_chunks_.size(); ++seq) {
upper_neighbor_chunks_[seq] = broker_->get_chunk(
HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR, seq);
if (!upper_neighbor_chunks_[seq]) {
LOG_ERROR("Missing hnsw streamer index chunk %zu th of %zu", seq,
upper_neighbor_chunks_.size());
return IndexError_InvalidFormat;
}
}
return 0;
}
int HnswRabitqStreamerEntity::open(IndexStorage::Pointer stg,
uint64_t max_index_size, bool check_crc) {
std::lock_guard<std::mutex> lock(mutex_);
bool huge_page = stg->isHugePage();
LOG_DEBUG("huge_page: %d", (int)huge_page);
int ret = init_chunk_params(max_index_size, huge_page);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("init_chunk_params failed for %s", IndexError::What(ret));
return ret;
}
ret = broker_->open(std::move(stg), max_index_size_, chunk_size_, check_crc);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Open index failed for %s", IndexError::What(ret));
return ret;
}
ret = upper_neighbor_index_->init(broker_, upper_neighbor_chunk_size_,
scaling_factor(), estimate_doc_capacity(),
kUpperHashMemoryInflateRatio);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Init neighbor hash map failed");
return ret;
}
//! init header
auto header_chunk =
broker_->get_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_HEADER,
HnswRabitqChunkBroker::kDefaultChunkSeqId);
if (!header_chunk) { // open empty index, create one
auto p = broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_HEADER,
HnswRabitqChunkBroker::kDefaultChunkSeqId,
header_size());
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc header chunk failed");
return p.first;
}
size_t size = p.second->write(0UL, &header(), header_size());
if (ailego_unlikely(size != header_size())) {
LOG_ERROR("Write header chunk failed");
return IndexError_WriteData;
}
return 0;
}
//! Open an exist hnsw index
ret = init_chunks(header_chunk);
if (ailego_unlikely(ret != 0)) {
return ret;
}
//! total docs including features wrote in index but neighbors may not ready
node_id_t total_vecs = 0;
if (node_chunks_.size() > 0) {
size_t last_idx = node_chunks_.size() - 1;
auto last_chunk = node_chunks_[last_idx];
if (last_chunk->data_size() % node_size()) {
LOG_WARN("The index may broken");
return IndexError_InvalidFormat;
}
total_vecs = last_idx * node_cnt_per_chunk_ +
node_chunks_[last_idx]->data_size() / node_size();
}
LOG_INFO(
"Open index, l0NeighborCnt=%zu upperNeighborCnt=%zu "
"efConstruction=%zu curDocCnt=%u totalVecs=%u maxLevel=%u",
l0_neighbor_cnt(), upper_neighbor_cnt(), ef_construction(), doc_cnt(),
total_vecs, cur_max_level());
//! try to correct the docCnt if index not fully flushed
if (doc_cnt() != total_vecs) {
LOG_WARN("Index closed abnormally, using totalVecs as curDocCnt");
*mutable_doc_cnt() = total_vecs;
}
if (filter_same_key_ || get_vector_enabled_) {
if (use_key_info_map_) {
for (node_id_t id = 0U; id < doc_cnt(); ++id) {
if (get_key(id) == kInvalidKey) {
continue;
}
(*keys_map_)[get_key(id)] = id;
}
}
}
stats_.set_loaded_count(doc_cnt());
return 0;
}
int HnswRabitqStreamerEntity::close() {
LOG_DEBUG("close index");
std::lock_guard<std::mutex> lock(mutex_);
flush_header();
mutable_header()->reset();
upper_neighbor_index_->cleanup();
keys_map_->clear();
header_.clear();
node_chunks_.clear();
upper_neighbor_chunks_.clear();
return broker_->close();
}
int HnswRabitqStreamerEntity::flush(uint64_t checkpoint) {
LOG_INFO("Flush index, curDocs=%zu", static_cast<size_t>(doc_cnt()));
std::lock_guard<std::mutex> lock(mutex_);
flush_header();
int ret = broker_->flush(checkpoint);
if (ret != 0) {
return ret;
}
return 0;
}
int HnswRabitqStreamerEntity::dump(const IndexDumper::Pointer &dumper) {
LOG_INFO("Dump index, curDocs=%zu", static_cast<size_t>(doc_cnt()));
//! sort by keys, to support get_vector by key in searcher
std::vector<key_t> keys(doc_cnt());
for (node_id_t i = 0; i < doc_cnt(); ++i) {
keys[i] = get_key(i);
}
//! dump neighbors
auto get_level = [&](node_id_t id) {
auto it = upper_neighbor_index_->find(id);
if (it == upper_neighbor_index_->end()) {
return 0U;
};
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
return meta->level;
};
auto ret = dump_segments(dumper, keys.data(), get_level);
if (ailego_unlikely(ret < 0)) {
return ret;
}
*stats_.mutable_dumped_size() += ret;
return 0;
}
int HnswRabitqStreamerEntity::check_hnsw_index(const HNSWHeader *hd) const {
if (l0_neighbor_cnt() != hd->l0_neighbor_cnt() ||
upper_neighbor_cnt() != hd->upper_neighbor_cnt()) {
LOG_ERROR("Param neighbor cnt: %zu:%zu mismatch index previous %zu:%zu",
l0_neighbor_cnt(), upper_neighbor_cnt(), hd->l0_neighbor_cnt(),
hd->upper_neighbor_cnt());
return IndexError_Mismatch;
}
if (vector_size() != hd->vector_size()) {
LOG_ERROR("vector size %zu mismatch index previous %zu", vector_size(),
hd->vector_size());
return IndexError_Mismatch;
}
if (ef_construction() != hd->ef_construction()) {
LOG_WARN("Param efConstruction %zu mismatch index previous %zu",
ef_construction(), hd->ef_construction());
}
if (scaling_factor() != hd->scaling_factor()) {
LOG_WARN("Param scalingFactor %zu mismatch index previous %zu",
scaling_factor(), hd->scaling_factor());
return IndexError_Mismatch;
}
if (prune_cnt() != hd->neighbor_prune_cnt()) {
LOG_WARN("Param pruneCnt %zu mismatch index previous %zu", prune_cnt(),
hd->neighbor_prune_cnt());
return IndexError_Mismatch;
}
if ((hd->entry_point() != kInvalidNodeId &&
hd->entry_point() >= hd->doc_cnt()) ||
(hd->entry_point() == kInvalidNodeId && hd->doc_cnt() > 0U)) {
LOG_WARN("Invalid entryPoint %zu, docCnt %zu",
static_cast<size_t>(hd->entry_point()),
static_cast<size_t>(hd->doc_cnt()));
return IndexError_InvalidFormat;
}
if (hd->entry_point() == kInvalidNodeId &&
broker_->get_chunk_cnt(HnswRabitqChunkBroker::CHUNK_TYPE_NODE) > 0) {
LOG_WARN("The index is broken, maybe it haven't flush");
return IndexError_InvalidFormat;
}
return 0;
}
int HnswRabitqStreamerEntity::add_vector(level_t level, key_t key,
const void *vec, node_id_t *id) {
Chunk::Pointer node_chunk;
size_t chunk_offset = -1UL;
std::lock_guard<std::mutex> lock(mutex_);
// duplicate check
if (ailego_unlikely(filter_same_key_ && get_id(key) != kInvalidNodeId)) {
LOG_WARN("Try to add duplicate key, ignore it");
return IndexError_Duplicate;
}
node_id_t local_id = static_cast<node_id_t>(doc_cnt());
uint32_t chunk_index = node_chunks_.size() - 1U;
if (chunk_index == -1U ||
(node_chunks_[chunk_index]->data_size() >=
node_cnt_per_chunk_ * node_size())) { // no space left and need to alloc
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
LOG_ERROR("add vector failed for no memory quota");
return IndexError_IndexFull;
}
chunk_index++;
auto p = broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_NODE,
chunk_index, chunk_size_);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return p.first;
}
node_chunk = p.second;
chunk_offset = 0UL;
node_chunks_.emplace_back(node_chunk);
} else {
node_chunk = node_chunks_[chunk_index];
chunk_offset = node_chunk->data_size();
}
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
if (ailego_unlikely(size != vector_size())) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
if (ailego_unlikely(size != sizeof(key_t))) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
//! level 0 neighbors is inited to zero by default
int ret = add_upper_neighbor(level, local_id);
if (ret != 0) {
return ret;
}
chunk_offset += node_size();
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
return IndexError_Runtime;
}
if (filter_same_key_ || get_vector_enabled_) {
if (use_key_info_map_) {
keys_map_lock_->lock();
(*keys_map_)[key] = local_id;
keys_map_lock_->unlock();
}
}
*mutable_doc_cnt() += 1;
broker_->mark_dirty();
*id = local_id;
return 0;
}
int HnswRabitqStreamerEntity::add_vector_with_id(level_t level, node_id_t id,
const void *vec) {
Chunk::Pointer node_chunk;
size_t chunk_offset = -1UL;
key_t key = id;
std::lock_guard<std::mutex> lock(mutex_);
// duplicate check
if (ailego_unlikely(filter_same_key_ && get_id(key) != kInvalidNodeId)) {
LOG_WARN("Try to add duplicate key, ignore it");
return IndexError_Duplicate;
}
// set node_chunk & chunk_offset if succeed
auto func_get_node_chunk_and_offset = [&](node_id_t node_id) -> int {
uint32_t chunk_index = node_id >> node_index_mask_bits_;
ailego_assert_with(chunk_index <= node_chunks_.size(), "invalid chunk idx");
// belongs to next chunk
if (chunk_index == node_chunks_.size()) {
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
LOG_ERROR("add vector failed for no memory quota");
return IndexError_IndexFull;
}
auto p = broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_NODE,
chunk_index, chunk_size_);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return p.first;
}
node_chunk = p.second;
node_chunks_.emplace_back(node_chunk);
}
node_chunk = node_chunks_[chunk_index];
chunk_offset = (node_id & node_index_mask_) * node_size();
return 0;
};
for (size_t start_id = doc_cnt(); start_id < id; ++start_id) {
if (auto ret = func_get_node_chunk_and_offset(start_id); ret != 0) {
LOG_ERROR("func_get_node_chunk_and_offset failed");
return ret;
}
size_t size = node_chunk->write(chunk_offset + vector_size(), &kInvalidKey,
sizeof(key_t));
if (ailego_unlikely(size != sizeof(key_t))) {
LOG_ERROR("Chunk write key failed, ret=%zu", size);
return IndexError_WriteData;
}
chunk_offset += node_size();
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
return IndexError_Runtime;
}
}
if (auto ret = func_get_node_chunk_and_offset(id); ret != 0) {
LOG_ERROR("func_get_node_chunk_and_offset failed");
return ret;
}
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
if (ailego_unlikely(size != vector_size())) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
if (ailego_unlikely(size != sizeof(key_t))) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
//! level 0 neighbors is inited to zero by default
int ret = add_upper_neighbor(level, id);
if (ret != 0) {
return ret;
}
if (*mutable_doc_cnt() <= id) {
*mutable_doc_cnt() = id + 1;
chunk_offset += node_size();
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
return IndexError_Runtime;
}
}
if (filter_same_key_ || get_vector_enabled_) {
if (use_key_info_map_) {
keys_map_lock_->lock();
(*keys_map_)[key] = id;
keys_map_lock_->unlock();
}
}
broker_->mark_dirty();
return 0;
}
void HnswRabitqStreamerEntity::update_ep_and_level(node_id_t ep,
level_t level) {
HnswRabitqEntity::update_ep_and_level(ep, level);
flush_header();
return;
}
const HnswRabitqEntity::Pointer HnswRabitqStreamerEntity::clone() const {
std::vector<Chunk::Pointer> node_chunks;
node_chunks.reserve(node_chunks_.size());
for (size_t i = 0UL; i < node_chunks_.size(); ++i) {
node_chunks.emplace_back(node_chunks_[i]->clone());
if (ailego_unlikely(!node_chunks[i])) {
LOG_ERROR("HnswRabitqStreamerEntity get chunk failed in clone");
return HnswRabitqEntity::Pointer();
}
}
std::vector<Chunk::Pointer> upper_neighbor_chunks;
upper_neighbor_chunks.reserve(upper_neighbor_chunks_.size());
for (size_t i = 0UL; i < upper_neighbor_chunks_.size(); ++i) {
upper_neighbor_chunks.emplace_back(upper_neighbor_chunks_[i]->clone());
if (ailego_unlikely(!upper_neighbor_chunks[i])) {
LOG_ERROR("HnswRabitqStreamerEntity get chunk failed in clone");
return HnswRabitqEntity::Pointer();
}
}
HnswRabitqStreamerEntity *entity =
new (std::nothrow) HnswRabitqStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_,
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswRabitqStreamerEntity new failed");
}
return HnswRabitqEntity::Pointer(entity);
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,522 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <iostream>
#include <ailego/parallel/lock.h>
#include <sparsehash/dense_hash_map>
#include <sparsehash/dense_hash_set>
#include <zvec/ailego/container/heap.h>
#include "zvec/core/framework/index_framework.h"
#include "hnsw_rabitq_chunk.h"
#include "hnsw_rabitq_entity.h"
#include "hnsw_rabitq_index_hash.h"
#include "hnsw_rabitq_params.h"
namespace zvec {
namespace core {
//! HnswRabitqStreamerEntity manage vector data, pkey, and node's neighbors
class HnswRabitqStreamerEntity : public HnswRabitqEntity {
public:
//! Cleanup
//! return 0 on success, or errCode in failure
virtual int cleanup() override;
//! Make a copy of streamer entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswRabitqEntity::Pointer clone() const override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
//! Get vector feature data by key
virtual const void *get_vector(node_id_t id) const override;
//! Get vectors feature data by local ids
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
//! Get the node id's neighbors on graph level
//! Note: the neighbors cannot be modified, using the following
//! method to get WritableNeighbors if want to
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
//! Add vector and key to hnsw entity, and local id will be saved in id
virtual int add_vector(level_t level, key_t key, const void *vec,
node_id_t *id) override;
//! Add vector and id to hnsw entity
virtual int add_vector_with_id(level_t level, node_id_t id,
const void *vec) override;
virtual int update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors)
override;
//! Append neighbor_id to node id neighbors on level
//! Notice: the caller must be ensure the neighbors not full
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
//! Dump index by dumper
virtual int dump(const IndexDumper::Pointer &dumper) override;
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
void set_use_key_info_map(bool use_id_map) {
use_key_info_map_ = use_id_map;
LOG_DEBUG("use_key_info_map_: %d", (int)use_key_info_map_);
}
public:
//! Constructor
HnswRabitqStreamerEntity(IndexStreamer::Stats &stats);
//! Destructor
~HnswRabitqStreamerEntity();
//! Get vector feature data by key
virtual const void *get_vector_by_key(key_t key) const override {
auto id = get_id(key);
return id == kInvalidNodeId ? nullptr : get_vector(id);
}
virtual int get_vector_by_key(
const key_t key, IndexStorage::MemoryBlock &block) const override {
auto id = get_id(key);
if (id != kInvalidNodeId) {
return get_vector(id, block);
} else {
return IndexError_InvalidArgument;
}
}
//! Init entity
int init(size_t max_doc_cnt);
//! Flush graph entity to disk
//! return 0 on success, or errCode in failure
int flush(uint64_t checkpoint);
//! Open entity from storage
//! return 0 on success, or errCode in failure
int open(IndexStorage::Pointer stg, uint64_t max_index_size, bool check_crc);
//! Close entity
//! return 0 on success, or errCode in failure
int close();
//! Set meta information from entity
int set_index_meta(const IndexMeta &meta) const {
return IndexHelper::SerializeToStorage(meta, broker_->storage().get());
}
//! Get meta information from entity
int get_index_meta(IndexMeta *meta) const {
return IndexHelper::DeserializeFromStorage(broker_->storage().get(), meta);
}
//! Set params: chunk size
inline void set_chunk_size(size_t val) {
chunk_size_ = val;
}
//! Set params
inline void set_filter_same_key(bool val) {
filter_same_key_ = val;
}
//! Set params
inline void set_get_vector(bool val) {
get_vector_enabled_ = val;
}
//! Get vector local id by key
inline node_id_t get_id(key_t key) const {
if (use_key_info_map_) {
keys_map_lock_->lock_shared();
auto it = keys_map_->find(key);
keys_map_lock_->unlock_shared();
return it == keys_map_->end() ? kInvalidNodeId : it->second;
} else {
return key;
}
}
void print_key_map() const {
std::cout << "key map begins" << std::endl;
auto iter = keys_map_->begin();
while (iter != keys_map_->end()) {
std::cout << "key: " << iter->first << ", id: " << iter->second
<< std::endl;
;
iter++;
}
std::cout << "key map ends" << std::endl;
}
//! Get l0 neighbors size
inline size_t neighbors_size() const {
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
}
//! Get neighbors size for level > 0
inline size_t upper_neighbors_size() const {
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
}
private:
union UpperNeighborIndexMeta {
struct {
uint32_t level : 4;
uint32_t index : 28; // index is composite type: chunk idx, and the
// N th neighbors in chunk, they two composite
// the 28 bits location
};
uint32_t data;
};
template <class Key, class T>
using HashMap = google::dense_hash_map<Key, T, std::hash<Key>>;
template <class Key, class T>
using HashMapPointer = std::shared_ptr<HashMap<Key, T>>;
template <class Key>
using HashSet = google::dense_hash_set<Key, std::hash<Key>>;
template <class Key>
using HashSetPointer = std::shared_ptr<HashSet<Key>>;
//! upper neighbor index hashmap
using NIHashMap = HnswIndexHashMap<node_id_t, uint32_t>;
using NIHashMapPointer = std::shared_ptr<NIHashMap>;
//! Private construct, only be called by clone method
HnswRabitqStreamerEntity(IndexStreamer::Stats &stats, const HNSWHeader &hd,
size_t chunk_size, uint32_t node_index_mask_bits,
uint32_t upper_neighbor_mask_bits,
bool filter_same_key, bool get_vector_enabled,
const NIHashMapPointer &upper_neighbor_index,
std::shared_ptr<ailego::SharedMutex> &keys_map_lock,
const HashMapPointer<key_t, node_id_t> &keys_map,
bool use_key_info_map,
std::vector<Chunk::Pointer> &&node_chunks,
std::vector<Chunk::Pointer> &&upper_neighbor_chunks,
const HnswRabitqChunkBroker::Pointer &broker)
: stats_(stats),
chunk_size_(chunk_size),
node_index_mask_bits_(node_index_mask_bits),
node_cnt_per_chunk_(1UL << node_index_mask_bits_),
node_index_mask_(node_cnt_per_chunk_ - 1),
upper_neighbor_mask_bits_(upper_neighbor_mask_bits),
upper_neighbor_mask_((1U << upper_neighbor_mask_bits_) - 1),
filter_same_key_(filter_same_key),
get_vector_enabled_(get_vector_enabled),
use_key_info_map_(use_key_info_map),
upper_neighbor_index_(upper_neighbor_index),
keys_map_lock_(keys_map_lock),
keys_map_(keys_map),
node_chunks_(std::move(node_chunks)),
upper_neighbor_chunks_(std::move(upper_neighbor_chunks)),
broker_(broker) {
*mutable_header() = hd;
neighbor_size_ = neighbors_size();
upper_neighbor_size_ = upper_neighbors_size();
}
//! Called only in searching procedure per context, so no need to lock
void sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE type, size_t idx,
std::vector<Chunk::Pointer> *chunks) const {
if (ailego_likely(idx < chunks->size())) {
return;
}
for (size_t i = chunks->size(); i <= idx; ++i) {
auto chunk = broker_->get_chunk(type, i);
// the storage can ensure get chunk will success after the first get
ailego_assert_with(!!chunk, "get chunk failed");
chunks->emplace_back(std::move(chunk));
}
}
//! return pair: chunk index + chunk offset
inline std::pair<uint32_t, uint32_t> get_vector_chunk_loc(
node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size();
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
&node_chunks_);
return std::make_pair(chunk_idx, offset);
}
//! return pair: chunk index + chunk offset
inline std::pair<uint32_t, uint32_t> get_key_chunk_loc(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
&node_chunks_);
return std::make_pair(chunk_idx, offset);
}
inline std::pair<uint32_t, uint32_t> get_upper_neighbor_chunk_loc(
level_t level, node_id_t id) const {
auto it = upper_neighbor_index_->find(id);
ailego_assert_abort(it != upper_neighbor_index_->end(),
"Get upper neighbor header failed");
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
uint32_t chunk_idx = (meta->index) >> upper_neighbor_mask_bits_;
uint32_t offset = (((meta->index) & upper_neighbor_mask_) + level - 1) *
upper_neighbor_size_;
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR, chunk_idx,
&upper_neighbor_chunks_);
ailego_assert_abort(chunk_idx < upper_neighbor_chunks_.size(),
"invalid chunk idx");
ailego_assert_abort(offset < upper_neighbor_chunks_[chunk_idx]->data_size(),
"invalid chunk offset");
return std::make_pair(chunk_idx, offset);
}
//! return pair: chunk + chunk offset
inline std::pair<Chunk *, size_t> get_neighbor_chunk_loc(level_t level,
node_id_t id) const {
if (level == 0UL) {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
&node_chunks_);
ailego_assert_abort(chunk_idx < node_chunks_.size(), "invalid chunk idx");
ailego_assert_abort(offset < node_chunks_[chunk_idx]->data_size(),
"invalid chunk offset");
return std::make_pair(node_chunks_[chunk_idx].get(), offset);
} else {
auto p = get_upper_neighbor_chunk_loc(level, id);
return std::make_pair(upper_neighbor_chunks_[p.first].get(), p.second);
}
}
//! Chunk hnsw index valid
int check_hnsw_index(const HNSWHeader *hd) const;
size_t get_total_upper_neighbors_size(level_t level) const {
return level * upper_neighbor_size_;
}
//! Add upper neighbor header and reserve space for upper neighbor
int add_upper_neighbor(level_t level, node_id_t id) {
if (level == 0) {
return 0;
}
Chunk::Pointer chunk;
uint64_t chunk_offset = -1UL;
size_t neighbors_size = get_total_upper_neighbors_size(level);
uint64_t chunk_index = upper_neighbor_chunks_.size() - 1UL;
if (chunk_index == -1UL ||
(upper_neighbor_chunks_[chunk_index]->padding_size() <
neighbors_size)) { // no space left and need to alloc
chunk_index++;
if (ailego_unlikely(upper_neighbor_chunks_.capacity() ==
upper_neighbor_chunks_.size())) {
LOG_ERROR("add upper neighbor failed for no memory quota");
return IndexError_IndexFull;
}
auto p =
broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR,
chunk_index, upper_neighbor_chunk_size_);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return p.first;
}
chunk = p.second;
chunk_offset = 0UL;
upper_neighbor_chunks_.emplace_back(chunk);
} else {
chunk = upper_neighbor_chunks_[chunk_index];
chunk_offset = chunk->data_size();
}
ailego_assert_with((size_t)level < kMaxGraphLayers, "invalid level");
ailego_assert_with(chunk_offset % upper_neighbor_size_ == 0,
"invalid offset");
ailego_assert_with((chunk_offset / upper_neighbor_size_) <
(1U << upper_neighbor_mask_bits_),
"invalid offset");
ailego_assert_with(chunk_index < (1U << (28 - upper_neighbor_mask_bits_)),
"invalid chunk index");
UpperNeighborIndexMeta meta;
meta.level = level;
meta.index = (chunk_index << upper_neighbor_mask_bits_) |
(chunk_offset / upper_neighbor_size_);
chunk_offset += upper_neighbor_size_ * level;
if (ailego_unlikely(!upper_neighbor_index_->insert(id, meta.data))) {
LOG_ERROR("HashMap insert value failed");
return IndexError_Runtime;
}
if (ailego_unlikely(chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", (size_t)chunk_offset);
return IndexError_Runtime;
}
return 0;
}
size_t estimate_doc_capacity() const {
return node_chunks_.capacity() * node_cnt_per_chunk_;
}
int init_chunk_params(size_t max_index_size, bool huge_page) {
node_cnt_per_chunk_ = std::max<uint32_t>(1, chunk_size_ / node_size());
//! align node cnt per chunk to pow of 2
node_index_mask_bits_ = std::ceil(std::log2(node_cnt_per_chunk_));
node_cnt_per_chunk_ = 1UL << node_index_mask_bits_;
if (huge_page) {
chunk_size_ = AlignHugePageSize(node_cnt_per_chunk_ * node_size());
} else {
chunk_size_ = AlignPageSize(node_cnt_per_chunk_ * node_size());
}
node_index_mask_ = node_cnt_per_chunk_ - 1;
if (max_index_size == 0UL) {
max_index_size_ = chunk_size_ * kDefaultMaxChunkCnt;
} else {
max_index_size_ = max_index_size;
}
//! To get a balanced upper neighbor chunk size.
//! If the upper chunk size is equal to node chunk size, it may waste
//! upper neighbor chunk space; if the upper neighbor chunk size is too
//! small, the will need large upper neighbor chunks index space. So to
//! get a balanced ratio be sqrt of the node/neighbor size ratio
float ratio =
std::sqrt(node_size() * scaling_factor() * 1.0f / upper_neighbor_size_);
if (huge_page) {
upper_neighbor_chunk_size_ = AlignHugePageSize(
std::max(get_total_upper_neighbors_size(kMaxGraphLayers),
static_cast<size_t>(chunk_size_ / ratio)));
} else {
upper_neighbor_chunk_size_ = AlignPageSize(
std::max(get_total_upper_neighbors_size(kMaxGraphLayers),
static_cast<size_t>(chunk_size_ / ratio)));
}
upper_neighbor_mask_bits_ =
std::ceil(std::log2(upper_neighbor_chunk_size_ / upper_neighbor_size_));
upper_neighbor_mask_ = (1 << upper_neighbor_mask_bits_) - 1;
size_t max_node_chunk_cnt = std::ceil(max_index_size_ / chunk_size_);
size_t max_upper_chunk_cnt = std::ceil(
(max_node_chunk_cnt * node_cnt_per_chunk_ * 1.0f / scaling_factor()) /
(upper_neighbor_chunk_size_ / upper_neighbor_size_));
max_upper_chunk_cnt =
max_upper_chunk_cnt + std::ceil(max_upper_chunk_cnt / scaling_factor());
//! reserve space to avoid memmove in chunks vector emplace chunk, so
//! as to lock-free in reading chunk
node_chunks_.reserve(max_node_chunk_cnt);
upper_neighbor_chunks_.reserve(max_upper_chunk_cnt);
LOG_DEBUG(
"Settings: nodeSize=%zu chunkSize=%u upperNeighborSize=%u "
"upperNeighborChunkSize=%u "
"nodeCntPerChunk=%u maxChunkCnt=%zu maxNeighborChunkCnt=%zu "
"maxIndexSize=%zu ratio=%.3f",
node_size(), chunk_size_, upper_neighbor_size_,
upper_neighbor_chunk_size_, node_cnt_per_chunk_, max_node_chunk_cnt,
max_upper_chunk_cnt, max_index_size_, ratio);
return 0;
}
//! Init node chunk and neighbor chunks
int init_chunks(const Chunk::Pointer &header_chunk);
int flush_header(void) {
if (!broker_->dirty()) {
// do not need to flush
return 0;
}
auto header_chunk =
broker_->get_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_HEADER,
HnswRabitqChunkBroker::kDefaultChunkSeqId);
if (ailego_unlikely(!header_chunk)) {
LOG_ERROR("get header chunk failed");
return IndexError_Runtime;
}
size_t size = header_chunk->write(0UL, &header(), header_size());
if (ailego_unlikely(size != header_size())) {
LOG_ERROR("Write header chunk failed");
return IndexError_WriteData;
}
return 0;
}
private:
HnswRabitqStreamerEntity(const HnswRabitqStreamerEntity &) = delete;
HnswRabitqStreamerEntity &operator=(const HnswRabitqStreamerEntity &) =
delete;
static constexpr uint64_t kUpperHashMemoryInflateRatio = 2.0f;
private:
IndexStreamer::Stats &stats_;
HNSWHeader header_{};
std::mutex mutex_{};
size_t max_index_size_{0UL};
uint32_t chunk_size_{kDefaultChunkSize};
uint32_t upper_neighbor_chunk_size_{kDefaultChunkSize};
uint32_t node_index_mask_bits_{0U};
uint32_t node_cnt_per_chunk_{0U};
uint32_t node_index_mask_{0U};
uint32_t neighbor_size_{0U};
uint32_t upper_neighbor_size_{0U};
//! UpperNeighborIndex.index composite chunkIdx and offset in chunk by the
//! following mask
uint32_t upper_neighbor_mask_bits_{0U};
uint32_t upper_neighbor_mask_{0U};
bool filter_same_key_{false};
bool get_vector_enabled_{false};
bool use_key_info_map_{true};
NIHashMapPointer upper_neighbor_index_{};
mutable std::shared_ptr<ailego::SharedMutex> keys_map_lock_{};
HashMapPointer<key_t, node_id_t> keys_map_{};
//! the chunks will be changed in searcher, so need mutable
//! data chunk include: vector, key, level 0 neighbors
mutable std::vector<Chunk::Pointer> node_chunks_{};
//! upper neighbor chunk inlude: UpperNeighborHeader + (1~level) neighbors
mutable std::vector<Chunk::Pointer> upper_neighbor_chunks_{};
HnswRabitqChunkBroker::Pointer broker_{}; // chunk broker
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,315 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "rabitq_converter.h"
#include <cstring>
#include <memory>
#include <rabitqlib/utils/rotator.hpp>
#include <zvec/ailego/container/params.h>
#include <zvec/ailego/parallel/thread_pool.h>
#include <zvec/ailego/utility/string_helper.h>
#include "ailego/pattern/defer.h"
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
#include "zvec/core/framework/index_cluster.h"
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_factory.h"
#include "zvec/core/framework/index_features.h"
#include "zvec/core/framework/index_holder.h"
#include "zvec/core/framework/index_memory.h"
#include "zvec/core/framework/index_meta.h"
#include "rabitq_params.h"
#include "rabitq_utils.h"
#ifdef _MSC_VER
#define strncasecmp _strnicmp
#endif
namespace zvec {
namespace core {
RabitqConverter::~RabitqConverter() {
this->cleanup();
}
int RabitqConverter::init(const IndexMeta &meta, const ailego::Params &params) {
// Copy meta and ensure it has metric information
meta_ = meta;
dimension_ = meta.dimension();
if (meta_.metric_name().empty()) {
LOG_ERROR("Meta metric is empty");
return IndexError_InvalidArgument;
}
// Round up dimension to multiple of 64
padded_dim_ = ((dimension_ + 63) / 64) * 64;
// Get RaBitQ parameters with defaults
uint32_t total_bits = 0;
params.get(PARAM_RABITQ_TOTAL_BITS, &total_bits);
if (total_bits == 0) {
total_bits = kDefaultRabitqTotalBits;
}
if (total_bits < 1 || total_bits > 9) {
LOG_ERROR("Invalid total_bits: %zu, must be in [1, 9]", (size_t)total_bits);
return IndexError_InvalidArgument;
}
ex_bits_ = total_bits - 1;
params.get(PARAM_RABITQ_NUM_CLUSTERS, &num_clusters_);
if (num_clusters_ == 0) {
num_clusters_ = kDefaultNumClusters;
}
if (ex_bits_ > 8) {
LOG_ERROR("Invalid ex_bits: %zu, must be <= 8", ex_bits_);
return IndexError_InvalidArgument;
}
if (meta.data_type() != IndexMeta::DataType::DT_FP32) {
LOG_ERROR("RaBitQ only supports FP32 data type");
return IndexError_Unsupported;
}
params.get(PARAM_RABITQ_SAMPLE_COUNT, &sample_count_);
std::string rotator_type_str;
params.get(PARAM_RABITQ_ROTATOR_TYPE, &rotator_type_str);
if (rotator_type_str.empty()) {
rotator_type_ = rabitqlib::RotatorType::FhtKacRotator;
} else if (strncasecmp(rotator_type_str.c_str(), "fht", 3) == 0) {
rotator_type_ = rabitqlib::RotatorType::FhtKacRotator;
} else if (strncasecmp(rotator_type_str.c_str(), "matrix", 6) == 0) {
rotator_type_ = rabitqlib::RotatorType::MatrixRotator;
} else {
LOG_ERROR("Invalid rotator_type: %s", rotator_type_str.c_str());
return IndexError_InvalidArgument;
}
// Create rotator
rotator_.reset(
rabitqlib::choose_rotator<float>(dimension_, rotator_type_, padded_dim_));
LOG_INFO(
"RabitqConverter initialized: dim=%zu, padded_dim=%zu, "
"num_clusters=%zu, ex_bits=%zu, rotator_type=%d[%s] sample_count[%zu]",
dimension_, padded_dim_, num_clusters_, ex_bits_, (int)rotator_type_,
rotator_type_str.c_str(), sample_count_);
return 0;
}
int RabitqConverter::cleanup() {
centroids_.clear();
rotated_centroids_.clear();
result_holder_.reset();
rotator_.reset();
return 0;
}
int RabitqConverter::train(IndexHolder::Pointer holder) {
if (!holder) {
LOG_ERROR("Null holder for training");
return IndexError_InvalidArgument;
}
ailego::ElapsedTime timer;
size_t vector_count = holder->count();
if (vector_count == 0) {
LOG_ERROR("No vectors for training");
return IndexError_InvalidArgument;
}
// do sampling from all data
size_t sample_count = vector_count;
if (sample_count_ > 0) {
sample_count = std::min(sample_count_, vector_count);
}
LOG_INFO("Training with %zu vectors from %zu of holder", sample_count,
vector_count);
auto sampler = std::make_shared<SampleIndexFeatures<CompactIndexFeatures>>(
meta_, sample_count);
auto iter = holder->create_iterator();
if (!iter) {
LOG_ERROR("Create iterator error");
return IndexError_Runtime;
}
for (; iter->is_valid(); iter->next()) {
sampler->emplace(iter->data());
}
// Holder is not needed, cleanup it.
holder.reset();
if (sampler->count() == 0) {
LOG_ERROR("Load training data error");
return IndexError_InvalidLength;
}
// Create KmeansCluster for training centroids
auto cluster = IndexFactory::CreateCluster("OptKmeansCluster");
if (!cluster) {
LOG_ERROR("Failed to create OptKmeansCluster");
return IndexError_NoExist;
}
// Initialize cluster
LOG_INFO(
"Initializing KmeansCluster with meta: dim=%u, data_type=%d, metric=%s",
meta_.dimension(), (int)meta_.data_type(), meta_.metric_name().c_str());
ailego::Params cluster_params;
int ret = cluster->init(meta_, cluster_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize KmeansCluster: %d", ret);
return ret;
}
ret = cluster->mount(sampler);
if (ret != 0) {
LOG_ERROR("Failed to mount training data: %d", ret);
return ret;
}
cluster->suggest(num_clusters_);
// Perform clustering
IndexCluster::CentroidList cents;
// TODO: support specify threads with argument
auto threads = std::make_shared<SingleQueueIndexThreads>(0, false);
ret = cluster->cluster(threads, cents);
if (ret != 0) {
LOG_ERROR("Failed to perform clustering: %d", ret);
return ret;
}
if (cents.size() != num_clusters_) {
LOG_WARN("Expected %zu clusters, got %zu", num_clusters_, cents.size());
num_clusters_ = cents.size();
}
// Extract original centroids (for LinearSeeker query)
centroids_.resize(num_clusters_ * dimension_);
// Extract rotated centroids (for quantization)
rotated_centroids_.resize(num_clusters_ * padded_dim_);
for (uint32_t i = 0; i < num_clusters_; ++i) {
const float *cent_data = static_cast<const float *>(cents[i].feature());
// Save original centroids
std::memcpy(&centroids_[i * dimension_], cent_data,
dimension_ * sizeof(float));
// Save rotated centroids
this->rotator_->rotate(cent_data, &rotated_centroids_[i * padded_dim_]);
}
stats_.set_trained_count(sampler->count());
stats_.set_trained_costtime(timer.milli_seconds());
LOG_INFO("Training completed: %zu centroids, cost %zu ms", num_clusters_,
static_cast<size_t>(timer.milli_seconds()));
return 0;
}
int RabitqConverter::transform(IndexHolder::Pointer holder) {
if (!holder) {
LOG_ERROR("Null holder for transformation");
return IndexError_InvalidArgument;
}
if (rotated_centroids_.empty()) {
LOG_ERROR("Centroids not trained yet");
return IndexError_NoReady;
}
LOG_ERROR("Not implemented");
return IndexError_NotImplemented;
}
int RabitqConverter::dump(const IndexDumper::Pointer &dumper) {
if (!dumper) {
LOG_ERROR("Null dumper");
return IndexError_InvalidArgument;
}
if (rotated_centroids_.empty() || centroids_.empty()) {
LOG_ERROR("No centroids to dump");
return IndexError_NoReady;
}
ailego::ElapsedTime timer;
size_t dumped_size = 0;
int ret = dump_rabitq_centroids(
dumper, dimension_, padded_dim_, ex_bits_, num_clusters_, rotator_type_,
rotated_centroids_, centroids_, rotator_, &dumped_size);
if (ret != 0) {
return ret;
}
stats_.set_dumped_size(dumped_size);
stats_.set_dumped_costtime(timer.milli_seconds());
LOG_INFO("Dump completed: %zu bytes, cost %zu ms", stats_.dumped_size(),
static_cast<size_t>(timer.milli_seconds()));
return 0;
}
int RabitqConverter::to_reformer(IndexReformer::Pointer *reformer) {
auto memory_dumper = IndexFactory::CreateDumper("MemoryDumper");
memory_dumper->init(ailego::Params());
std::string file_id = ailego::StringHelper::Concat(
"rabitq_converter_", ailego::Monotime::MilliSeconds(), rand());
int ret = memory_dumper->create(file_id);
if (ret != 0) {
LOG_ERROR("Failed to create memory dumper: %d", ret);
return ret;
}
// Release memory
AILEGO_DEFER([&file_id]() { IndexMemory::Instance()->remove(file_id); });
ret = this->dump(memory_dumper);
if (ret != 0) {
LOG_ERROR("Failed to dump RabitqConverter: %d", ret);
return ret;
}
ret = memory_dumper->close();
if (ret != 0) {
LOG_ERROR("Failed to close memory dumper: %d", ret);
return ret;
}
auto res = std::make_shared<RabitqReformer>();
ailego::Params reformer_params;
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
ret = res->init(reformer_params);
if (ret != 0) {
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
return ret;
}
auto memory_storage = IndexFactory::CreateStorage("MemoryReadStorage");
ret = memory_storage->open(file_id, false);
if (ret != 0) {
LOG_ERROR("Failed to open memory storage: %d", ret);
return ret;
}
ret = res->load(memory_storage);
if (ret != 0) {
LOG_ERROR("Failed to load RabitqReformer: %d", ret);
return ret;
}
*reformer = std::move(res);
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,101 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <vector>
#include <rabitqlib/utils/rotator.hpp>
#include "zvec/core/framework/index_cluster.h"
#include "zvec/core/framework/index_converter.h"
#include "zvec/core/framework/index_reformer.h"
#include "zvec/core/framework/index_threads.h"
#include "rabitq_params.h"
namespace zvec {
namespace core {
class RabitqReformer;
/*! RaBitQ Converter
* Trains KMeans centroids and quantizes vectors using RaBitQ
*/
class RabitqConverter : public IndexConverter {
public:
//! Constructor
RabitqConverter() = default;
//! Destructor
~RabitqConverter() override;
//! Initialize Converter
int init(const IndexMeta &meta, const ailego::Params &params) override;
//! Cleanup Converter
int cleanup(void) override;
//! Train the data - perform KMeans clustering
int train(IndexHolder::Pointer holder) override;
//! Transform the data - quantize vectors using RaBitQ
int transform(IndexHolder::Pointer holder) override;
//! Dump centroids and config into storage
int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
const Stats &stats(void) const override {
return stats_;
}
//! Retrieve a holder as result
IndexHolder::Pointer result(void) const override {
return result_holder_;
}
//! Retrieve Index Meta
const IndexMeta &meta(void) const override {
return meta_;
}
int to_reformer(IndexReformer::Pointer *reformer) override;
private:
static inline size_t AlignSize(size_t size) {
return (size + 0x1F) & (~0x1F);
}
private:
IndexMeta meta_;
IndexHolder::Pointer result_holder_;
Stats stats_;
size_t sample_count_{0};
// RaBitQ parameters
size_t num_clusters_{0};
size_t ex_bits_{0};
size_t dimension_{0};
size_t padded_dim_{0};
// Original centroids: num_clusters * dimension (for LinearSeeker query)
std::vector<float> centroids_;
// Rotated centroids: num_clusters * padded_dim (for quantization)
std::vector<float> rotated_centroids_;
// Rotator for vector transformation
rabitqlib::RotatorType rotator_type_{rabitqlib::RotatorType::FhtKacRotator};
std::unique_ptr<rabitqlib::Rotator<float>> rotator_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,49 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
namespace zvec {
namespace core {
// Local metric type enum that mirrors rabitqlib::MetricType,
// without exposing rabitqlib headers to consumers of this file.
enum class RabitqMetricType {
kL2 = 0,
kIP = 1,
};
// RaBitQ Converter parameters
static const std::string PARAM_RABITQ_NUM_CLUSTERS(
"proxima.rabitq.num_clusters");
static const std::string PARAM_RABITQ_TOTAL_BITS("proxima.rabitq.total_bits");
static const std::string PARAM_RABITQ_METRIC_NAME("proxima.rabitq.metric_name");
static const std::string PARAM_RABITQ_ROTATOR_TYPE(
"proxima.rabitq.rotator.type");
static const std::string PARAM_RABITQ_SAMPLE_COUNT(
"proxima.rabitq.sample_count");
// Default values
constexpr size_t kDefaultNumClusters = 16;
// 4-bit, 5-bit, and 7-bit quantization typically achieve 90%, 95%, and 99%
// recall, respectively—without accessing raw vectors for reranking
constexpr size_t kDefaultRabitqTotalBits = 7;
constexpr int kMinRabitqDimSize = 64;
constexpr int kMaxRabitqDimSize = 4095;
} // namespace core
} // namespace zvec
@@ -0,0 +1,467 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "rabitq_reformer.h"
#include <string>
#include <vector>
#include <rabitqlib/defines.hpp>
#include <rabitqlib/index/query.hpp>
#include <rabitqlib/quantization/rabitq.hpp>
#include <rabitqlib/utils/rotator.hpp>
#include <zvec/ailego/logger/logger.h>
#include <zvec/ailego/utility/string_helper.h>
#include "core/algorithm/cluster/linear_seeker.h"
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_factory.h"
#include "zvec/core/framework/index_features.h"
#include "zvec/core/framework/index_meta.h"
#include "zvec/core/framework/index_storage.h"
#include "hnsw_rabitq_query_entity.h"
#include "rabitq_converter.h"
#include "rabitq_utils.h"
namespace zvec {
namespace core {
// All rabitqlib types are confined to this translation unit via pimpl.
struct RabitqReformer::Impl {
// RaBitQ parameters
size_t num_clusters{0};
size_t ex_bits{0};
size_t dimension{0};
size_t padded_dim{0};
size_t size_bin_data{0};
size_t size_ex_data{0};
bool loaded{false};
// Original centroids: num_clusters * dimension (for LinearSeeker query)
std::vector<float> centroids;
// Rotated centroids: num_clusters * padded_dim (for quantization)
std::vector<float> rotated_centroids;
rabitqlib::RotatorType rotator_type{rabitqlib::RotatorType::FhtKacRotator};
std::unique_ptr<rabitqlib::Rotator<float>> rotator;
rabitqlib::quant::RabitqConfig query_config;
rabitqlib::quant::RabitqConfig config;
rabitqlib::MetricType metric_type{rabitqlib::METRIC_L2};
LinearSeeker::Pointer centroid_seeker;
CoherentIndexFeatures::Pointer centroid_features;
// Translate local enum to rabitqlib enum (used only inside this .cc).
static rabitqlib::MetricType to_rabitq(RabitqMetricType m) {
return m == RabitqMetricType::kIP ? rabitqlib::METRIC_IP
: rabitqlib::METRIC_L2;
}
// Translate rabitqlib enum to local enum.
static RabitqMetricType from_rabitq(rabitqlib::MetricType m) {
return m == rabitqlib::METRIC_IP ? RabitqMetricType::kIP
: RabitqMetricType::kL2;
}
int quantize_vector(const float *raw_vector, uint32_t cluster_id,
std::string *quantized_data) const;
};
RabitqReformer::RabitqReformer() : impl_(std::make_unique<Impl>()) {}
RabitqReformer::~RabitqReformer() {
this->cleanup();
}
size_t RabitqReformer::num_clusters() const {
return impl_->num_clusters;
}
RabitqMetricType RabitqReformer::rabitq_metric_type() const {
return Impl::from_rabitq(impl_->metric_type);
}
int RabitqReformer::init(const ailego::Params &params) {
std::string metric_name = params.get_as_string(PARAM_RABITQ_METRIC_NAME);
if (metric_name == "SquaredEuclidean") {
impl_->metric_type = rabitqlib::METRIC_L2;
} else if (metric_name == "InnerProduct") {
impl_->metric_type = rabitqlib::METRIC_IP;
} else if (metric_name == "Cosine") {
impl_->metric_type = rabitqlib::METRIC_IP;
} else {
LOG_ERROR("Unsupported metric name: %s", metric_name.c_str());
return IndexError_InvalidArgument;
}
LOG_DEBUG("Rabitq reformer init done. metric_name=%s metric_type=%d",
metric_name.c_str(), static_cast<int>(impl_->metric_type));
return 0;
}
int RabitqReformer::cleanup() {
impl_->centroids.clear();
impl_->rotated_centroids.clear();
impl_->centroid_seeker.reset();
impl_->centroid_features.reset();
impl_->loaded = false;
impl_->rotator.reset();
return 0;
}
int RabitqReformer::unload() {
return this->cleanup();
}
int RabitqReformer::load(IndexStorage::Pointer storage) {
if (!storage) {
LOG_ERROR("Invalid storage for load");
return IndexError_InvalidArgument;
}
auto segment = storage->get(RABITQ_CONVERTER_SEG_ID);
if (!segment) {
LOG_ERROR("Failed to get segment %s", RABITQ_CONVERTER_SEG_ID.c_str());
return IndexError_InvalidFormat;
}
size_t offset = 0;
RabitqConverterHeader header;
IndexStorage::MemoryBlock block;
size_t size = segment->read(offset, block, sizeof(header));
if (size != sizeof(header)) {
LOG_ERROR("Failed to read header");
return IndexError_InvalidFormat;
}
memcpy(&header, block.data(), sizeof(header));
impl_->dimension = header.dim;
impl_->padded_dim = header.padded_dim;
impl_->ex_bits = header.ex_bits;
impl_->num_clusters = header.num_clusters;
impl_->rotator_type =
static_cast<rabitqlib::RotatorType>(header.rotator_type);
offset += sizeof(header);
// Read rotated centroids
size_t rotated_centroids_size =
sizeof(float) * header.num_clusters * header.padded_dim;
size = segment->read(offset, block, rotated_centroids_size);
if (size != rotated_centroids_size) {
LOG_ERROR("Failed to read rotated centroids");
return IndexError_InvalidFormat;
}
impl_->rotated_centroids.resize(header.num_clusters * header.padded_dim);
memcpy(impl_->rotated_centroids.data(), block.data(), rotated_centroids_size);
offset += size;
// Read original centroids (for LinearSeeker query)
size_t centroids_size = sizeof(float) * header.num_clusters * header.dim;
size = segment->read(offset, block, centroids_size);
if (size != centroids_size) {
LOG_ERROR("Failed to read centroids");
return IndexError_InvalidFormat;
}
impl_->centroids.resize(header.num_clusters * header.dim);
memcpy(impl_->centroids.data(), block.data(), centroids_size);
offset += size;
// Read rotator
size_t rotator_size = header.rotator_size;
size = segment->read(offset, block, rotator_size);
if (size != rotator_size) {
LOG_ERROR("Failed to read rotator");
return IndexError_InvalidFormat;
}
impl_->rotator.reset(rabitqlib::choose_rotator<float>(
impl_->dimension, impl_->rotator_type, impl_->padded_dim));
impl_->rotator->load(reinterpret_cast<const char *>(block.data()));
offset += size;
impl_->query_config = rabitqlib::quant::faster_config(
impl_->padded_dim, rabitqlib::SplitSingleQuery<float>::kNumBits);
impl_->config =
rabitqlib::quant::faster_config(impl_->padded_dim, impl_->ex_bits + 1);
impl_->size_bin_data =
rabitqlib::BinDataMap<float>::data_bytes(impl_->padded_dim);
impl_->size_ex_data = rabitqlib::ExDataMap<float>::data_bytes(
impl_->padded_dim, impl_->ex_bits);
// Initialize LinearSeeker for centroid search
IndexMeta centroid_meta;
centroid_meta.set_data_type(IndexMeta::DataType::DT_FP32);
centroid_meta.set_dimension(static_cast<uint32_t>(impl_->dimension));
// Note:
// 1. spherical kmeans is used for InnerProduct and Cosine, so centroids are
// normalized.
// 2. for Cosine metric, `transform_to_entity` input is normalized, need to
// use InnerProduct metric as Cosine metric requires extra dimension which is
// unsuitable for centroids.
centroid_meta.set_metric(impl_->metric_type == rabitqlib::METRIC_L2
? "SquaredEuclidean"
: "InnerProduct",
0, ailego::Params());
impl_->centroid_features = std::make_shared<CoherentIndexFeatures>();
impl_->centroid_features->mount(centroid_meta, impl_->centroids.data(),
impl_->centroids.size() * sizeof(float));
impl_->centroid_seeker = std::make_shared<LinearSeeker>();
int ret = impl_->centroid_seeker->init(centroid_meta);
if (ret != 0) {
LOG_ERROR("Failed to init centroid seeker. ret[%d]", ret);
return ret;
}
ret = impl_->centroid_seeker->mount(impl_->centroid_features);
if (ret != 0) {
LOG_ERROR("Failed to mount centroid features. ret[%d]", ret);
return ret;
}
LOG_INFO(
"Rabitq reformer load done. dimension=%zu, padded_dim=%zu, "
"ex_bits=%zu, num_clusters=%zu, size_bin_data=%zu, size_ex_data=%zu "
"rotator_type=%d",
impl_->dimension, impl_->padded_dim, impl_->ex_bits, impl_->num_clusters,
impl_->size_bin_data, impl_->size_ex_data, (int)impl_->rotator_type);
impl_->loaded = true;
return 0;
}
int RabitqReformer::convert(const void *record, const IndexQueryMeta &rmeta,
std::string *out, IndexQueryMeta *ometa) const {
if (!impl_->loaded) {
LOG_ERROR("Centroids not loaded yet");
return IndexError_NoReady;
}
if (!record || !out) {
LOG_ERROR("Invalid arguments for convert");
return IndexError_InvalidArgument;
}
// input may be transformed, require rmeta.dimension >= dimension
if (rmeta.dimension() < impl_->dimension ||
rmeta.data_type() != IndexMeta::DataType::DT_FP32) {
LOG_ERROR("Invalid record meta: dimension=%zu, data_type=%d",
static_cast<size_t>(rmeta.dimension()), (int)rmeta.data_type());
return IndexError_InvalidArgument;
}
// Find nearest centroid using LinearSeeker
Seeker::Document doc;
int ret = impl_->centroid_seeker->seek(
record, impl_->dimension * sizeof(float), &doc);
if (ret != 0) {
LOG_ERROR("Failed to seek centroid. ret[%d]", ret);
return ret;
}
uint32_t cluster_id = doc.index;
const float *vector = static_cast<const float *>(record);
ret = impl_->quantize_vector(vector, cluster_id, out);
if (ret != 0) {
LOG_ERROR("Failed to quantize vector");
return ret;
}
ometa->set_meta(IndexMeta::DataType::DT_INT8, (uint32_t)out->size());
return 0;
}
int RabitqReformer::transform(const void *, const IndexQueryMeta &,
std::string *, IndexQueryMeta *) const {
return IndexError_NotImplemented;
}
int RabitqReformer::transform_to_entity(const void *query,
HnswRabitqQueryEntity *entity) const {
if (!impl_->loaded) {
LOG_ERROR("Centroids not loaded yet");
return IndexError_NoReady;
}
if (!query) {
LOG_ERROR("Invalid arguments for transform");
return IndexError_InvalidArgument;
}
const float *query_vector = static_cast<const float *>(query);
// Apply rotator
entity->rotated_query.resize(impl_->padded_dim);
impl_->rotator->rotate(query_vector, entity->rotated_query.data());
// Quantize query to 4-bit representation
entity->query_wrapper = std::make_unique<rabitqlib::SplitSingleQuery<float>>(
entity->rotated_query.data(), impl_->padded_dim, impl_->ex_bits,
impl_->query_config, impl_->metric_type);
// Preprocess - get the distance from query to all centroids
entity->q_to_centroids.resize(impl_->num_clusters);
if (impl_->metric_type == rabitqlib::METRIC_L2) {
for (size_t i = 0; i < impl_->num_clusters; i++) {
entity->q_to_centroids[i] = std::sqrt(rabitqlib::euclidean_sqr(
entity->rotated_query.data(),
impl_->rotated_centroids.data() + (i * impl_->padded_dim),
impl_->padded_dim));
}
} else if (impl_->metric_type == rabitqlib::METRIC_IP) {
entity->q_to_centroids.resize(impl_->num_clusters * 2);
// first half as g_add, second half as g_error
for (size_t i = 0; i < impl_->num_clusters; i++) {
entity->q_to_centroids[i] = rabitqlib::dot_product(
entity->rotated_query.data(),
impl_->rotated_centroids.data() + (i * impl_->padded_dim),
impl_->padded_dim);
entity->q_to_centroids[i + impl_->num_clusters] =
std::sqrt(rabitqlib::euclidean_sqr(
entity->rotated_query.data(),
impl_->rotated_centroids.data() + (i * impl_->padded_dim),
impl_->padded_dim));
}
}
return 0;
}
int RabitqReformer::Impl::quantize_vector(const float *raw_vector,
uint32_t cluster_id,
std::string *quantized_data) const {
std::vector<float> rotated_data(padded_dim);
rotator->rotate(raw_vector, rotated_data.data());
// quantized format: cluster_id + bin_data + ex_data
quantized_data->resize(sizeof(cluster_id) + size_bin_data + size_ex_data);
memcpy(&(*quantized_data)[0], &cluster_id, sizeof(cluster_id));
int bin_data_offset = sizeof(cluster_id);
int ex_data_offset = bin_data_offset + size_bin_data;
rabitqlib::quant::quantize_split_single(
rotated_data.data(), rotated_centroids.data() + (cluster_id * padded_dim),
padded_dim, ex_bits, &(*quantized_data)[bin_data_offset],
&(*quantized_data)[ex_data_offset], metric_type, config);
return 0;
}
int RabitqReformer::dump(const IndexDumper::Pointer &dumper) {
if (!dumper) {
LOG_ERROR("Null dumper");
return IndexError_InvalidArgument;
}
if (!impl_->loaded || impl_->rotated_centroids.empty() ||
impl_->centroids.empty()) {
LOG_ERROR("No centroids to dump");
return IndexError_NoReady;
}
size_t dumped_size = 0;
int ret = dump_rabitq_centroids(
dumper, impl_->dimension, impl_->padded_dim, impl_->ex_bits,
impl_->num_clusters, impl_->rotator_type, impl_->rotated_centroids,
impl_->centroids, impl_->rotator, &dumped_size);
if (ret != 0) {
return ret;
}
LOG_INFO("RabitqReformer dump completed: %zu bytes", dumped_size);
return 0;
}
int RabitqReformer::dump(const IndexStorage::Pointer &storage) {
if (!storage) {
LOG_ERROR("Null storage");
return IndexError_InvalidArgument;
}
if (!impl_->loaded || impl_->rotated_centroids.empty() ||
impl_->centroids.empty()) {
LOG_ERROR("No centroids to dump");
return IndexError_NoReady;
}
auto align_size = [](size_t size) -> size_t {
return (size + 0x1F) & (~0x1F);
};
size_t header_size = sizeof(RabitqConverterHeader);
size_t rotated_centroids_size =
impl_->rotated_centroids.size() * sizeof(float);
size_t centroids_size = impl_->centroids.size() * sizeof(float);
size_t rotator_size = impl_->rotator->dump_bytes();
size_t data_size =
header_size + rotated_centroids_size + centroids_size + rotator_size;
size_t total_size = align_size(data_size);
int ret = storage->append(RABITQ_CONVERTER_SEG_ID, total_size);
if (ret != 0) {
LOG_ERROR("Failed to append segment %s, ret=%d",
RABITQ_CONVERTER_SEG_ID.c_str(), ret);
return ret;
}
auto segment = storage->get(RABITQ_CONVERTER_SEG_ID);
if (!segment) {
LOG_ERROR("Failed to get segment %s", RABITQ_CONVERTER_SEG_ID.c_str());
return IndexError_ReadData;
}
size_t offset = 0;
RabitqConverterHeader header;
header.dim = static_cast<uint32_t>(impl_->dimension);
header.padded_dim = static_cast<uint32_t>(impl_->padded_dim);
header.num_clusters = static_cast<uint32_t>(impl_->num_clusters);
header.ex_bits = static_cast<uint8_t>(impl_->ex_bits);
header.rotator_type = static_cast<uint8_t>(impl_->rotator_type);
header.rotator_size = static_cast<uint32_t>(rotator_size);
size_t written = segment->write(offset, &header, header_size);
if (written != header_size) {
LOG_ERROR("Failed to write header: written=%zu, expected=%zu", written,
header_size);
return IndexError_WriteData;
}
offset += header_size;
written = segment->write(offset, impl_->rotated_centroids.data(),
rotated_centroids_size);
if (written != rotated_centroids_size) {
LOG_ERROR("Failed to write rotated centroids: written=%zu, expected=%zu",
written, rotated_centroids_size);
return IndexError_WriteData;
}
offset += rotated_centroids_size;
written = segment->write(offset, impl_->centroids.data(), centroids_size);
if (written != centroids_size) {
LOG_ERROR("Failed to write centroids: written=%zu, expected=%zu", written,
centroids_size);
return IndexError_WriteData;
}
offset += centroids_size;
std::vector<char> buffer(rotator_size);
impl_->rotator->save(buffer.data());
written = segment->write(offset, buffer.data(), rotator_size);
if (written != rotator_size) {
LOG_ERROR("Failed to write rotator data: written=%zu, expected=%zu",
written, rotator_size);
return IndexError_WriteData;
}
LOG_INFO("RabitqReformer dump to storage completed: %zu bytes", data_size);
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,72 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#pragma once
#include <memory>
#include "zvec/core/framework/index_dumper.h"
#include "zvec/core/framework/index_reformer.h"
#include "zvec/core/framework/index_storage.h"
#include "rabitq_params.h"
namespace zvec {
namespace core {
class HnswRabitqQueryEntity;
/*! RaBitQ Reformer
* Loads centroids and performs query transformation and vector quantization.
*
* All rabitqlib types are hidden behind a pimpl to avoid leaking rabitqlib
* headers to consumers of this class.
*/
class RabitqReformer : public IndexReformer {
public:
typedef std::shared_ptr<RabitqReformer> Pointer;
RabitqReformer();
~RabitqReformer() override;
// Non-copyable
RabitqReformer(const RabitqReformer &) = delete;
RabitqReformer &operator=(const RabitqReformer &) = delete;
int init(const ailego::Params &params) override;
int cleanup(void) override;
int load(IndexStorage::Pointer storage) override;
int unload(void) override;
// transform() is not implemented for RabitqReformer; use transform_to_entity.
int transform(const void *query, const IndexQueryMeta &qmeta,
std::string *out, IndexQueryMeta *ometa) const override;
int convert(const void *record, const IndexQueryMeta &rmeta, std::string *out,
IndexQueryMeta *ometa) const override;
int dump(const IndexDumper::Pointer &dumper);
int dump(const IndexStorage::Pointer &storage);
int transform_to_entity(const void *query,
HnswRabitqQueryEntity *entity) const;
size_t num_clusters() const;
RabitqMetricType rabitq_metric_type() const;
private:
struct Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,115 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "rabitq_utils.h"
#include <string>
#include <zvec/ailego/hash/crc32c.h>
#include "zvec/core/framework/index_error.h"
#include "zvec/core/framework/index_logger.h"
namespace zvec {
namespace core {
int dump_rabitq_centroids(
const IndexDumper::Pointer &dumper, size_t dimension, size_t padded_dim,
size_t ex_bits, size_t num_clusters, rabitqlib::RotatorType rotator_type,
const std::vector<float> &rotated_centroids,
const std::vector<float> &centroids,
const std::unique_ptr<rabitqlib::Rotator<float>> &rotator,
size_t *out_dumped_size) {
auto align_size = [](size_t size) -> size_t {
return (size + 0x1F) & (~0x1F);
};
uint32_t crc = 0;
size_t dumped_size = 0;
// Write header
RabitqConverterHeader header;
header.dim = static_cast<uint32_t>(dimension);
header.padded_dim = static_cast<uint32_t>(padded_dim);
header.num_clusters = static_cast<uint32_t>(num_clusters);
header.ex_bits = static_cast<uint8_t>(ex_bits);
header.rotator_type = static_cast<uint8_t>(rotator_type);
header.rotator_size = static_cast<uint32_t>(rotator->dump_bytes());
size_t size = dumper->write(&header, sizeof(header));
if (size != sizeof(header)) {
LOG_ERROR("Failed to write header: written=%zu, expected=%zu", size,
sizeof(header));
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(&header, sizeof(header), crc);
dumped_size += size;
// Write rotated centroids
size = dumper->write(rotated_centroids.data(),
rotated_centroids.size() * sizeof(float));
if (size != rotated_centroids.size() * sizeof(float)) {
LOG_ERROR("Failed to write rotated centroids: written=%zu, expected=%zu",
size, rotated_centroids.size() * sizeof(float));
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(rotated_centroids.data(),
rotated_centroids.size() * sizeof(float), crc);
dumped_size += size;
// Write original centroids
size = dumper->write(centroids.data(), centroids.size() * sizeof(float));
if (size != centroids.size() * sizeof(float)) {
LOG_ERROR("Failed to write centroids: written=%zu, expected=%zu", size,
centroids.size() * sizeof(float));
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(centroids.data(), centroids.size() * sizeof(float),
crc);
dumped_size += size;
// Write rotator data
std::vector<char> buffer(rotator->dump_bytes());
rotator->save(buffer.data());
size = dumper->write(buffer.data(), buffer.size());
if (size != buffer.size()) {
LOG_ERROR("Failed to write rotator data: written=%zu, expected=%zu", size,
buffer.size());
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(buffer.data(), buffer.size(), crc);
dumped_size += size;
// Write padding
size_t padding_size = align_size(dumped_size) - dumped_size;
if (padding_size > 0) {
std::string padding(padding_size, '\0');
if (dumper->write(padding.data(), padding_size) != padding_size) {
LOG_ERROR("Append padding failed, size %lu", padding_size);
return IndexError_WriteData;
}
}
int ret =
dumper->append(RABITQ_CONVERTER_SEG_ID, dumped_size, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s meta failed, ret=%d",
RABITQ_CONVERTER_SEG_ID.c_str(), ret);
return ret;
}
if (out_dumped_size) {
*out_dumped_size = dumped_size;
}
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,53 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <vector>
#include <rabitqlib/utils/rotator.hpp>
#include "zvec/core/framework/index_dumper.h"
namespace zvec {
namespace core {
inline const std::string RABITQ_CONVERTER_SEG_ID{"rabitq.converter"};
struct RabitqConverterHeader {
uint32_t num_clusters;
uint32_t dim;
uint32_t padded_dim;
uint32_t rotator_size;
uint8_t ex_bits;
uint8_t rotator_type;
uint8_t padding[2];
uint32_t reserve[3];
RabitqConverterHeader() {
memset(this, 0, sizeof(RabitqConverterHeader));
}
};
static_assert(sizeof(RabitqConverterHeader) % 32 == 0,
"RabitqConverterHeader must be aligned with 32 bytes");
// Common dump implementation for RabitqConverter and RabitqReformer
int dump_rabitq_centroids(
const IndexDumper::Pointer &dumper, size_t dimension, size_t padded_dim,
size_t ex_bits, size_t num_clusters, rabitqlib::RotatorType rotator_type,
const std::vector<float> &rotated_centroids,
const std::vector<float> &centroids,
const std::unique_ptr<rabitqlib::Rotator<float>> &rotator,
size_t *out_dumped_size = nullptr);
} // namespace core
} // namespace zvec