feat: add hnsw-rabitq support (#69)
* feat: add hnsw-rabitq * undefine transform * support config rotator type * support sample_count * fix ut * refactor: update interface * refactor: update interface * update rabitq index params * fix interface update * fix searcher test * fix streamer test * streamer support bf and add more ut * rm env check * add collection ut * add schema check * add rabitq query param binding * add integration test * fix local_builder * cleanup dist calculator * add RaBitQ-Library submodule * cleanup rabitq converter/reformer * add files * disable build on mac * disable Feature_Optimize_HNSW_RABITQ * disable python/tests/test_collection_hnsw_rabitq.py:13 * check avx2/avx512 * fix refine * fix mac ci * add dimension check * fix mac ci * fix ci * add missing lib * fix centroids selection for Cosine/InnerProduct * rename hnsw-rabitq to hnsw_rabitq * address comments * fix compile * fix rabitqlib name * fix mac compile * check avx2/avx512 support * runtime check again compile-time * check AUTO_DETECT_ARCH * rm debug log * fix * rabitq use avx2 by default * add missing file * fix search_bf/group_by dist * address comments * fix typo * address comments * rabitq support disable id_map * address comments
This commit is contained in:
@@ -6,4 +6,25 @@ cc_directory(flat)
|
||||
cc_directory(flat_sparse)
|
||||
cc_directory(ivf)
|
||||
cc_directory(hnsw)
|
||||
cc_directory(hnsw_sparse)
|
||||
cc_directory(hnsw_sparse)
|
||||
if(RABITQ_SUPPORTED)
|
||||
message(STATUS "BUILD RABITQ")
|
||||
cc_directory(hnsw_rabitq)
|
||||
else()
|
||||
message(STATUS "NOT BUILD RABITQ")
|
||||
# Empty stub library for unsupported platforms
|
||||
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/rabitq_stub.cc
|
||||
"// Stub implementation for unsupported platforms\n"
|
||||
"// RaBitQ only supports Linux x86_64\n"
|
||||
"namespace zvec { namespace core { /* empty namespace for compatibility */ } }\n"
|
||||
)
|
||||
|
||||
cc_library(
|
||||
NAME core_knn_hnsw_rabitq
|
||||
STATIC SHARED STRICT ALWAYS_LINK
|
||||
SRCS ${CMAKE_CURRENT_BINARY_DIR}/rabitq_stub.cc
|
||||
LIBS core_framework
|
||||
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
endif()
|
||||
|
||||
@@ -1250,7 +1250,7 @@ int OptKmeansCluster::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
auto type_ = meta.data_type();
|
||||
|
||||
if (meta.metric_name() == "InnerProduct") {
|
||||
if (meta.metric_name() == "InnerProduct" || meta.metric_name() == "Cosine") {
|
||||
switch (type_) {
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
algorithm_.reset(
|
||||
|
||||
@@ -127,7 +127,6 @@ int FlatBuilder<BATCH_SIZE>::dump(const IndexDumper::Pointer &dumper) {
|
||||
return error_code;
|
||||
}
|
||||
|
||||
holder_ = nullptr;
|
||||
stats_.set_dumped_count(keys.size());
|
||||
stats_.set_dumped_costtime(stamp.milli_seconds());
|
||||
return 0;
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
|
||||
if(AUTO_DETECT_ARCH)
|
||||
foreach(FILE ${HNSW_RABITQ_FILES})
|
||||
set_source_files_properties(
|
||||
${FILE}
|
||||
PROPERTIES
|
||||
COMPILE_FLAGS "${RABITQ_ARCH_FLAG}"
|
||||
)
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
cc_library(
|
||||
NAME core_knn_hnsw_rabitq
|
||||
STATIC SHARED STRICT ALWAYS_LINK
|
||||
SRCS *.cc
|
||||
LIBS core_framework rabitqlib sparsehash
|
||||
INCS . ${PROJECT_ROOT_DIR}/src ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
@@ -0,0 +1,382 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_algorithm.h"
|
||||
#include <chrono>
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswRabitqAlgorithm::HnswRabitqAlgorithm(HnswRabitqEntity &entity)
|
||||
: entity_(entity),
|
||||
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
|
||||
lock_pool_(kLockCnt) {}
|
||||
|
||||
int HnswRabitqAlgorithm::cleanup() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqAlgorithm::add_node(node_id_t id, level_t level,
|
||||
HnswRabitqContext *ctx) {
|
||||
spin_lock_.lock();
|
||||
|
||||
// std::cout << "id: " << id << ", level: " << level << std::endl;
|
||||
|
||||
auto cur_max_level = entity_.cur_max_level();
|
||||
auto entry_point = entity_.entry_point();
|
||||
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
|
||||
entity_.update_ep_and_level(id, level);
|
||||
spin_lock_.unlock();
|
||||
return 0;
|
||||
}
|
||||
spin_lock_.unlock();
|
||||
|
||||
if (ailego_unlikely(level > cur_max_level)) {
|
||||
mutex_.lock();
|
||||
// re-check max level
|
||||
cur_max_level = entity_.cur_max_level();
|
||||
entry_point = entity_.entry_point();
|
||||
if (level <= cur_max_level) {
|
||||
mutex_.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
level_t cur_level = cur_max_level;
|
||||
ResultRecord dist = ctx->dist_calculator()(entry_point);
|
||||
for (; cur_level > level; --cur_level) {
|
||||
select_entry_point(cur_level, &entry_point, &dist, ctx);
|
||||
}
|
||||
|
||||
for (; cur_level >= 0; --cur_level) {
|
||||
search_neighbors(cur_level, &entry_point, &dist, ctx->level_topk(cur_level),
|
||||
ctx);
|
||||
}
|
||||
|
||||
// add neighbors from down level to top level, to avoid upper level visible
|
||||
// to knn_search but the under layer level not ready
|
||||
for (cur_level = 0; cur_level <= level; ++cur_level) {
|
||||
add_neighbors(id, cur_level, ctx->level_topk(cur_level), ctx);
|
||||
ctx->level_topk(cur_level).clear();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(level > cur_max_level)) {
|
||||
spin_lock_.lock();
|
||||
entity_.update_ep_and_level(id, level);
|
||||
spin_lock_.unlock();
|
||||
mutex_.unlock();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! select_entry_point on hnsw level, ef = 1
|
||||
void HnswRabitqAlgorithm::select_entry_point(level_t level,
|
||||
node_id_t *entry_point,
|
||||
ResultRecord *dist,
|
||||
HnswRabitqContext *ctx) const {
|
||||
auto &entity = ctx->get_entity();
|
||||
HnswRabitqAddDistCalculator &dc = ctx->dist_calculator();
|
||||
while (true) {
|
||||
const Neighbors neighbors = entity.get_neighbors(level, *entry_point);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
uint32_t size = neighbors.size();
|
||||
if (size == 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
|
||||
int ret = dc.get_vector(&neighbors[0], size, neighbor_vec_blocks);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_vector())++;
|
||||
}
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
break;
|
||||
}
|
||||
|
||||
bool find_closer = false;
|
||||
|
||||
std::vector<float> dists(size);
|
||||
std::vector<const void *> neighbor_vecs(size);
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
|
||||
}
|
||||
|
||||
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
ResultRecord cur_dist = dists[i];
|
||||
|
||||
if (cur_dist < *dist) {
|
||||
*entry_point = neighbors[i];
|
||||
*dist = cur_dist;
|
||||
find_closer = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!find_closer) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswRabitqAlgorithm::add_neighbors(node_id_t id, level_t level,
|
||||
TopkHeap &topk_heap,
|
||||
HnswRabitqContext *ctx) {
|
||||
if (ailego_unlikely(topk_heap.size() == 0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
HnswRabitqAddDistCalculator &dc = ctx->dist_calculator();
|
||||
|
||||
update_neighbors(dc, id, level, topk_heap);
|
||||
|
||||
// reverse update neighbors
|
||||
for (size_t i = 0; i < topk_heap.size(); ++i) {
|
||||
reverse_update_neighbors(dc, topk_heap[i].first, level, id,
|
||||
topk_heap[i].second, ctx->update_heap());
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswRabitqAlgorithm::search_neighbors(level_t level,
|
||||
node_id_t *entry_point,
|
||||
ResultRecord *dist, TopkHeap &topk,
|
||||
HnswRabitqContext *ctx) const {
|
||||
const auto &entity = ctx->get_entity();
|
||||
HnswRabitqAddDistCalculator &dc = ctx->dist_calculator();
|
||||
VisitFilter &visit = ctx->visit_filter();
|
||||
CandidateHeap &candidates = ctx->candidates();
|
||||
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
|
||||
if (ctx->filter().is_valid()) {
|
||||
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
|
||||
}
|
||||
|
||||
candidates.clear();
|
||||
visit.clear();
|
||||
visit.set_visited(*entry_point);
|
||||
if (!filter(*entry_point)) {
|
||||
topk.emplace(*entry_point, *dist);
|
||||
}
|
||||
|
||||
candidates.emplace(*entry_point, *dist);
|
||||
while (!candidates.empty() && !ctx->reach_scan_limit()) {
|
||||
auto top = candidates.begin();
|
||||
node_id_t main_node = top->first;
|
||||
ResultRecord main_dist = top->second;
|
||||
|
||||
if (topk.full() && main_dist > topk[0].second) {
|
||||
break;
|
||||
}
|
||||
|
||||
candidates.pop();
|
||||
const Neighbors neighbors = entity.get_neighbors(level, main_node);
|
||||
ailego_prefetch(neighbors.data);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
|
||||
std::vector<node_id_t> neighbor_ids(neighbors.size());
|
||||
uint32_t size = 0;
|
||||
for (uint32_t i = 0; i < neighbors.size(); ++i) {
|
||||
node_id_t node = neighbors[i];
|
||||
if (visit.visited(node)) {
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_visit_dup_cnt())++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
visit.set_visited(node);
|
||||
neighbor_ids[size++] = node;
|
||||
}
|
||||
if (size == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
|
||||
int ret = dc.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_vector())++;
|
||||
}
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
break;
|
||||
}
|
||||
|
||||
// do prefetch
|
||||
static constexpr node_id_t BATCH_SIZE = 12;
|
||||
static constexpr node_id_t PREFETCH_STEP = 2;
|
||||
for (uint32_t i = 0; i < std::min(BATCH_SIZE * PREFETCH_STEP, size); ++i) {
|
||||
ailego_prefetch(neighbor_vec_blocks[i].data());
|
||||
}
|
||||
// done
|
||||
|
||||
std::vector<float> dists(size);
|
||||
std::vector<const void *> neighbor_vecs(size);
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
|
||||
}
|
||||
|
||||
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
node_id_t node = neighbor_ids[i];
|
||||
ResultRecord cur_dist = dists[i];
|
||||
|
||||
if ((!topk.full()) || cur_dist < topk[0].second) {
|
||||
candidates.emplace(node, cur_dist);
|
||||
// update entry_point for next level scan
|
||||
if (cur_dist < *dist) {
|
||||
*entry_point = node;
|
||||
*dist = cur_dist;
|
||||
}
|
||||
if (!filter(node)) {
|
||||
topk.emplace(node, cur_dist);
|
||||
}
|
||||
} // end if
|
||||
} // end for
|
||||
} // while
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswRabitqAlgorithm::update_neighbors(HnswRabitqAddDistCalculator &dc,
|
||||
node_id_t id, level_t level,
|
||||
TopkHeap &topk_heap) {
|
||||
topk_heap.sort();
|
||||
|
||||
uint32_t max_neighbor_cnt = entity_.neighbor_cnt(level);
|
||||
if (topk_heap.size() <= static_cast<size_t>(entity_.prune_cnt())) {
|
||||
if (topk_heap.size() <= static_cast<size_t>(max_neighbor_cnt)) {
|
||||
entity_.update_neighbors(level, id, topk_heap);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t cur_size = 0;
|
||||
for (size_t i = 0; i < topk_heap.size(); ++i) {
|
||||
node_id_t cur_node = topk_heap[i].first;
|
||||
ResultRecord cur_node_dist = topk_heap[i].second;
|
||||
bool good = true;
|
||||
for (uint32_t j = 0; j < cur_size; ++j) {
|
||||
ResultRecord tmp_dist = dc.dist(cur_node, topk_heap[j].first);
|
||||
if (tmp_dist <= cur_node_dist) {
|
||||
good = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (good) {
|
||||
topk_heap[cur_size].first = cur_node;
|
||||
topk_heap[cur_size].second = cur_node_dist;
|
||||
cur_size++;
|
||||
if (cur_size >= max_neighbor_cnt) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// when after-prune neighbor count is too seldom,
|
||||
// we use this strategy to make-up enough edges
|
||||
// not only just make-up out-degrees
|
||||
// we also make-up enough in-degrees
|
||||
uint32_t min_neighbors = entity_.min_neighbor_cnt();
|
||||
for (size_t k = cur_size; cur_size < min_neighbors && k < topk_heap.size();
|
||||
++k) {
|
||||
bool exist = false;
|
||||
for (size_t j = 0; j < cur_size; ++j) {
|
||||
if (topk_heap[j].first == topk_heap[k].first) {
|
||||
exist = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!exist) {
|
||||
topk_heap[cur_size].first = topk_heap[k].first;
|
||||
topk_heap[cur_size].second = topk_heap[k].second;
|
||||
cur_size++;
|
||||
}
|
||||
}
|
||||
|
||||
topk_heap.resize(cur_size);
|
||||
entity_.update_neighbors(level, id, topk_heap);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswRabitqAlgorithm::reverse_update_neighbors(
|
||||
HnswRabitqAddDistCalculator &dc, node_id_t id, level_t level,
|
||||
node_id_t link_id, ResultRecord dist, TopkHeap &update_heap) {
|
||||
const size_t max_neighbor_cnt = entity_.neighbor_cnt(level);
|
||||
|
||||
uint32_t lock_idx = id & kLockMask;
|
||||
lock_pool_[lock_idx].lock();
|
||||
const Neighbors neighbors = entity_.get_neighbors(level, id);
|
||||
size_t size = neighbors.size();
|
||||
ailego_assert_with(size <= max_neighbor_cnt, "invalid neighbor size");
|
||||
if (size < max_neighbor_cnt) {
|
||||
entity_.add_neighbor(level, id, size, link_id);
|
||||
lock_pool_[lock_idx].unlock();
|
||||
return;
|
||||
}
|
||||
|
||||
update_heap.emplace(link_id, dist);
|
||||
|
||||
for (size_t i = 0; i < size; ++i) {
|
||||
node_id_t node = neighbors[i];
|
||||
ResultRecord cur_dist = dc.dist(id, node);
|
||||
update_heap.emplace(node, cur_dist);
|
||||
}
|
||||
|
||||
//! TODO: optimize prune
|
||||
//! prune edges
|
||||
update_heap.sort();
|
||||
size_t cur_size = 0;
|
||||
for (size_t i = 0; i < update_heap.size(); ++i) {
|
||||
node_id_t cur_node = update_heap[i].first;
|
||||
ResultRecord cur_node_dist = update_heap[i].second;
|
||||
bool good = true;
|
||||
for (size_t j = 0; j < cur_size; ++j) {
|
||||
ResultRecord tmp_dist = dc.dist(cur_node, update_heap[j].first);
|
||||
if (tmp_dist <= cur_node_dist) {
|
||||
good = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (good) {
|
||||
update_heap[cur_size].first = cur_node;
|
||||
update_heap[cur_size].second = cur_node_dist;
|
||||
cur_size++;
|
||||
if (cur_size >= max_neighbor_cnt) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
update_heap.resize(cur_size);
|
||||
entity_.update_neighbors(level, id, update_heap);
|
||||
|
||||
lock_pool_[lock_idx].unlock();
|
||||
|
||||
update_heap.clear();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,124 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include "hnsw_rabitq_context.h"
|
||||
#include "hnsw_rabitq_dist_calculator.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! hnsw graph algorithm implement
|
||||
class HnswRabitqAlgorithm {
|
||||
public:
|
||||
typedef std::unique_ptr<HnswRabitqAlgorithm> UPointer;
|
||||
|
||||
public:
|
||||
//! Constructor
|
||||
explicit HnswRabitqAlgorithm(HnswRabitqEntity &entity);
|
||||
|
||||
//! Destructor
|
||||
~HnswRabitqAlgorithm() = default;
|
||||
|
||||
//! Cleanup HnswRabitqAlgorithm
|
||||
int cleanup();
|
||||
|
||||
//! Add a node to hnsw graph
|
||||
//! @id: the node unique id
|
||||
//! @level: a node will be add to graph in each level [0, level]
|
||||
//! return 0 on success, or errCode in failure
|
||||
int add_node(node_id_t id, level_t level, HnswRabitqContext *ctx);
|
||||
|
||||
//! Initiate HnswRabitqAlgorithm
|
||||
int init() {
|
||||
level_probas_.clear();
|
||||
double level_mult =
|
||||
1 / std::log(static_cast<double>(entity_.scaling_factor()));
|
||||
for (int level = 0;; level++) {
|
||||
// refers faiss get_random_level alg
|
||||
double proba =
|
||||
std::exp(-level / level_mult) * (1 - std::exp(-1 / level_mult));
|
||||
if (proba < 1e-9) {
|
||||
break;
|
||||
}
|
||||
level_probas_.push_back(proba);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Generate a random level
|
||||
//! return graph level
|
||||
uint32_t get_random_level() const {
|
||||
// gen rand float (0, 1)
|
||||
double f = mt_() / static_cast<float>(mt_.max());
|
||||
for (size_t level = 0; level < level_probas_.size(); level++) {
|
||||
if (f < level_probas_[level]) {
|
||||
return level;
|
||||
}
|
||||
f -= level_probas_[level];
|
||||
}
|
||||
return level_probas_.size() - 1;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Select in upper layer to get entry point for next layer search
|
||||
void select_entry_point(level_t level, node_id_t *entry_point,
|
||||
ResultRecord *dist, HnswRabitqContext *ctx) const;
|
||||
|
||||
//! update node id neighbors from topkHeap, and reverse link is also updated
|
||||
void add_neighbors(node_id_t id, level_t level, TopkHeap &topk_heap,
|
||||
HnswRabitqContext *ctx);
|
||||
|
||||
//! Given a node id and level, search the nearest neighbors in graph
|
||||
//! Note: the nearest neighbors result keeps in topk, and entry_point and
|
||||
//! dist will be updated to current level nearest node id and distance
|
||||
void search_neighbors(level_t level, node_id_t *entry_point,
|
||||
ResultRecord *dist, TopkHeap &topk,
|
||||
HnswRabitqContext *ctx) const;
|
||||
|
||||
//! Update the node's neighbors
|
||||
void update_neighbors(HnswRabitqAddDistCalculator &dc, node_id_t id,
|
||||
level_t level, TopkHeap &topk_heap);
|
||||
|
||||
//! Checking linkId could be id's new neighbor, and add as neighbor if true
|
||||
//! @dc distance calculator
|
||||
//! @updateHeap temporary heap in updating neighbors
|
||||
void reverse_update_neighbors(HnswRabitqAddDistCalculator &dc, node_id_t id,
|
||||
level_t level, node_id_t link_id,
|
||||
ResultRecord dist, TopkHeap &update_heap);
|
||||
|
||||
private:
|
||||
HnswRabitqAlgorithm(const HnswRabitqAlgorithm &) = delete;
|
||||
HnswRabitqAlgorithm &operator=(const HnswRabitqAlgorithm &) = delete;
|
||||
|
||||
private:
|
||||
static constexpr uint32_t kLockCnt{1U << 8};
|
||||
static constexpr uint32_t kLockMask{kLockCnt - 1U};
|
||||
|
||||
HnswRabitqEntity &entity_;
|
||||
mutable std::mt19937 mt_{};
|
||||
std::vector<double> level_probas_{};
|
||||
|
||||
mutable ailego::SpinMutex spin_lock_{}; // global spin lock
|
||||
std::mutex mutex_{}; // global mutex
|
||||
// TODO: spin lock?
|
||||
std::vector<std::mutex> lock_pool_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,554 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_builder.h"
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
#include <ailego/pattern/defer.h>
|
||||
#include <zvec/ailego/container/params.h>
|
||||
#include <zvec/ailego/utility/string_helper.h>
|
||||
#include <zvec/ailego/utility/time_helper.h>
|
||||
#include "zvec/core/framework/index_error.h"
|
||||
#include "zvec/core/framework/index_factory.h"
|
||||
#include "zvec/core/framework/index_logger.h"
|
||||
#include "zvec/core/framework/index_memory.h"
|
||||
#include "zvec/core/framework/index_meta.h"
|
||||
#include "zvec/core/framework/index_provider.h"
|
||||
#include "hnsw_rabitq_algorithm.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
#include "hnsw_rabitq_params.h"
|
||||
#include "rabitq_converter.h"
|
||||
#include "rabitq_params.h"
|
||||
#include "rabitq_reformer.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswRabitqBuilder::HnswRabitqBuilder() {}
|
||||
|
||||
int HnswRabitqBuilder::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
LOG_INFO("Begin HnswRabitqBuilder::init");
|
||||
|
||||
meta_ = meta;
|
||||
auto params_copy = params;
|
||||
meta_.set_builder("HnswRabitqBuilder", HnswRabitqEntity::kRevision,
|
||||
std::move(params_copy));
|
||||
|
||||
size_t memory_quota = 0UL;
|
||||
params.get(PARAM_HNSW_RABITQ_BUILDER_MEMORY_QUOTA, &memory_quota);
|
||||
params.get(PARAM_HNSW_RABITQ_BUILDER_THREAD_COUNT, &thread_cnt_);
|
||||
params.get(PARAM_HNSW_RABITQ_BUILDER_MIN_NEIGHBOR_COUNT, &min_neighbor_cnt_);
|
||||
params.get(PARAM_HNSW_RABITQ_BUILDER_EFCONSTRUCTION, &ef_construction_);
|
||||
params.get(PARAM_HNSW_RABITQ_BUILDER_CHECK_INTERVAL_SECS,
|
||||
&check_interval_secs_);
|
||||
|
||||
params.get(PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT,
|
||||
&upper_max_neighbor_cnt_);
|
||||
float multiplier = HnswRabitqEntity::kDefaultL0MaxNeighborCntMultiplier;
|
||||
params.get(PARAM_HNSW_RABITQ_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER,
|
||||
&multiplier);
|
||||
l0_max_neighbor_cnt_ = multiplier * upper_max_neighbor_cnt_;
|
||||
scaling_factor_ = upper_max_neighbor_cnt_;
|
||||
params.get(PARAM_HNSW_RABITQ_BUILDER_SCALING_FACTOR, &scaling_factor_);
|
||||
|
||||
multiplier = HnswRabitqEntity::kDefaultNeighborPruneMultiplier;
|
||||
params.get(PARAM_HNSW_RABITQ_BUILDER_NEIGHBOR_PRUNE_MULTIPLIER, &multiplier);
|
||||
size_t prune_cnt = multiplier * upper_max_neighbor_cnt_;
|
||||
|
||||
if (ef_construction_ == 0) {
|
||||
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
|
||||
}
|
||||
if (upper_max_neighbor_cnt_ == 0) {
|
||||
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
|
||||
}
|
||||
if (upper_max_neighbor_cnt_ > kMaxNeighborCnt) {
|
||||
LOG_ERROR("[%s] must be in range (0,%d]",
|
||||
PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT.c_str(),
|
||||
kMaxNeighborCnt);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (min_neighbor_cnt_ > upper_max_neighbor_cnt_) {
|
||||
LOG_ERROR("[%s]-[%d] must be <= [%s]-[%d]",
|
||||
PARAM_HNSW_RABITQ_BUILDER_MIN_NEIGHBOR_COUNT.c_str(),
|
||||
min_neighbor_cnt_,
|
||||
PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT.c_str(),
|
||||
upper_max_neighbor_cnt_);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (l0_max_neighbor_cnt_ == 0) {
|
||||
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
|
||||
}
|
||||
if (l0_max_neighbor_cnt_ > HnswRabitqEntity::kMaxNeighborCnt) {
|
||||
LOG_ERROR("L0MaxNeighborCnt must be in range (0,%d)",
|
||||
HnswRabitqEntity::kMaxNeighborCnt);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (scaling_factor_ == 0U) {
|
||||
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
|
||||
}
|
||||
if (scaling_factor_ < 5 || scaling_factor_ > 1000) {
|
||||
LOG_ERROR("[%s] must be in range [5,1000]",
|
||||
PARAM_HNSW_RABITQ_BUILDER_SCALING_FACTOR.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (thread_cnt_ == 0) {
|
||||
thread_cnt_ = std::thread::hardware_concurrency();
|
||||
}
|
||||
if (thread_cnt_ > std::thread::hardware_concurrency()) {
|
||||
LOG_WARN("[%s] greater than cpu cores %zu",
|
||||
PARAM_HNSW_RABITQ_BUILDER_THREAD_COUNT.c_str(),
|
||||
static_cast<size_t>(std::thread::hardware_concurrency()));
|
||||
}
|
||||
if (prune_cnt == 0UL) {
|
||||
prune_cnt = upper_max_neighbor_cnt_;
|
||||
}
|
||||
|
||||
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
int ret = metric_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t total_bits = 0;
|
||||
params.get(PARAM_RABITQ_TOTAL_BITS, &total_bits);
|
||||
if (total_bits == 0) {
|
||||
total_bits = kDefaultRabitqTotalBits;
|
||||
}
|
||||
if (total_bits < 1 || total_bits > 9) {
|
||||
LOG_ERROR("Invalid total_bits: %zu, must be in [1, 9]", (size_t)total_bits);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
uint8_t ex_bits = total_bits - 1;
|
||||
entity_.set_ex_bits(ex_bits);
|
||||
|
||||
uint32_t dimension = 0;
|
||||
params.get(PARAM_HNSW_RABITQ_GENERAL_DIMENSION, &dimension);
|
||||
if (dimension == 0) {
|
||||
LOG_ERROR("%s not set", PARAM_HNSW_RABITQ_GENERAL_DIMENSION.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (dimension < kMinRabitqDimSize || dimension > kMaxRabitqDimSize) {
|
||||
LOG_ERROR("Invalid dimension: %u, must be in [%d, %d]", dimension,
|
||||
kMinRabitqDimSize, kMaxRabitqDimSize);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
entity_.update_rabitq_params_and_vector_size(dimension);
|
||||
|
||||
entity_.set_ef_construction(ef_construction_);
|
||||
entity_.set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
|
||||
entity_.set_min_neighbor_cnt(min_neighbor_cnt_);
|
||||
entity_.set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
|
||||
entity_.set_scaling_factor(scaling_factor_);
|
||||
entity_.set_memory_quota(memory_quota);
|
||||
entity_.set_prune_cnt(prune_cnt);
|
||||
|
||||
ret = entity_.init();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
alg_ = HnswRabitqAlgorithm::UPointer(new HnswRabitqAlgorithm(entity_));
|
||||
|
||||
ret = alg_->init();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Create and initialize RaBitQ converter
|
||||
converter_ = std::make_shared<RabitqConverter>();
|
||||
|
||||
IndexMeta converter_meta = meta_;
|
||||
converter_meta.set_dimension(dimension);
|
||||
ret = converter_->init(converter_meta, params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize RabitqConverter: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
state_ = BUILD_STATE_INITED;
|
||||
LOG_INFO(
|
||||
"End HnswRabitqBuilder::init, params: rawVectorSize=%u vectorSize=%zu "
|
||||
"efConstruction=%u "
|
||||
"l0NeighborCnt=%u upperNeighborCnt=%u scalingFactor=%u "
|
||||
"memoryQuota=%zu neighborPruneCnt=%zu metricName=%s ",
|
||||
meta_.element_size(), entity_.vector_size(), ef_construction_,
|
||||
l0_max_neighbor_cnt_, upper_max_neighbor_cnt_, scaling_factor_,
|
||||
memory_quota, prune_cnt, meta_.metric_name().c_str());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilder::cleanup(void) {
|
||||
LOG_INFO("Begin HnswRabitqBuilder::cleanup");
|
||||
|
||||
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultL0MaxNeighborCnt;
|
||||
min_neighbor_cnt_ = 0;
|
||||
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
|
||||
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
|
||||
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
|
||||
check_interval_secs_ = kDefaultLogIntervalSecs;
|
||||
errcode_ = 0;
|
||||
error_ = false;
|
||||
entity_.cleanup();
|
||||
if (alg_) {
|
||||
alg_->cleanup();
|
||||
}
|
||||
meta_.clear();
|
||||
metric_.reset();
|
||||
stats_.clear_attributes();
|
||||
stats_.set_trained_count(0UL);
|
||||
stats_.set_built_count(0UL);
|
||||
stats_.set_dumped_count(0UL);
|
||||
stats_.set_discarded_count(0UL);
|
||||
stats_.set_trained_costtime(0UL);
|
||||
stats_.set_built_costtime(0UL);
|
||||
stats_.set_dumped_costtime(0UL);
|
||||
state_ = BUILD_STATE_INIT;
|
||||
|
||||
LOG_INFO("End HnswRabitqBuilder::cleanup");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilder::train(IndexThreads::Pointer,
|
||||
IndexHolder::Pointer holder) {
|
||||
if (state_ != BUILD_STATE_INITED) {
|
||||
LOG_ERROR("Init the builder before HnswRabitqBuilder::train");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (!holder) {
|
||||
LOG_ERROR("Input holder is nullptr while training index");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (!holder->is_matched(meta_)) {
|
||||
LOG_ERROR("Input holder doesn't match index meta while training index");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
LOG_INFO("Begin HnswRabitqBuilder::train");
|
||||
size_t trained_cost_time = 0;
|
||||
size_t trained_count = 0;
|
||||
|
||||
int ret = train_converter_and_load_reformer(holder);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (metric_->support_train()) {
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
while (iter->is_valid()) {
|
||||
ret = metric_->train(iter->data(), meta_.dimension());
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw build measure train failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
iter->next();
|
||||
++trained_count;
|
||||
}
|
||||
trained_cost_time = ailego::Monotime::MilliSeconds() - start_time;
|
||||
}
|
||||
stats_.set_trained_count(trained_count);
|
||||
stats_.set_trained_costtime(trained_cost_time);
|
||||
state_ = BUILD_STATE_TRAINED;
|
||||
|
||||
LOG_INFO("End HnswRabitqBuilder::train");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilder::train_converter_and_load_reformer(
|
||||
IndexHolder::Pointer holder) {
|
||||
// Train converter (KMeans clustering)
|
||||
int ret = converter_->train(holder);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to train RabitqConverter: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
auto memory_dumper = IndexFactory::CreateDumper("MemoryDumper");
|
||||
memory_dumper->init(ailego::Params());
|
||||
std::string file_id = ailego::StringHelper::Concat(
|
||||
"rabitq_converter_", ailego::Monotime::MilliSeconds(), rand());
|
||||
ret = memory_dumper->create(file_id);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to create memory dumper: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
// Release memory
|
||||
AILEGO_DEFER([&file_id]() { IndexMemory::Instance()->remove(file_id); });
|
||||
ret = converter_->dump(memory_dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to dump RabitqConverter: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
ret = memory_dumper->close();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to close memory dumper: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
reformer_ = std::make_shared<RabitqReformer>();
|
||||
ailego::Params reformer_params;
|
||||
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
|
||||
ret = reformer_->init(reformer_params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
auto memory_storage = IndexFactory::CreateStorage("MemoryReadStorage");
|
||||
ret = memory_storage->open(file_id, false);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to open memory storage: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
ret = reformer_->load(memory_storage);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to load RabitqReformer: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilder::train(const IndexTrainer::Pointer & /*trainer*/) {
|
||||
if (state_ != BUILD_STATE_INITED) {
|
||||
LOG_ERROR("Init the builder before HnswRabitqBuilder::train");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Begin HnswRabitqBuilder::train by trainer");
|
||||
|
||||
stats_.set_trained_count(0UL);
|
||||
stats_.set_trained_costtime(0UL);
|
||||
state_ = BUILD_STATE_TRAINED;
|
||||
|
||||
LOG_INFO("End HnswRabitqBuilder::train by trainer");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilder::build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) {
|
||||
if (state_ != BUILD_STATE_TRAINED) {
|
||||
LOG_ERROR("Train the index before HnswRabitqBuilder::build");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (!holder) {
|
||||
LOG_ERROR("Input holder is nullptr while building index");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (!holder->is_matched(meta_)) {
|
||||
LOG_ERROR("Input holder doesn't match index meta while building index");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
IndexProvider::Pointer provider =
|
||||
std::dynamic_pointer_cast<IndexProvider>(holder);
|
||||
if (!provider) {
|
||||
LOG_ERROR("Rabitq builder expect IndexProvider");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_cnt_, false);
|
||||
}
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
LOG_INFO("Begin HnswRabitqBuilder::build");
|
||||
|
||||
if (holder->count() != static_cast<size_t>(-1)) {
|
||||
LOG_DEBUG("HnswRabitqBuilder holder documents count %lu", holder->count());
|
||||
int ret = entity_.reserve_space(holder->count());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("HnswBuilde reserver space failed");
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator for holder failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
int ret;
|
||||
error_ = false;
|
||||
IndexQueryMeta ometa;
|
||||
ometa.set_meta(holder->data_type(), holder->dimension());
|
||||
while (iter->is_valid()) {
|
||||
const void *vec = iter->data();
|
||||
// quantize vector
|
||||
std::string converted_vector;
|
||||
IndexQueryMeta converted_meta;
|
||||
ret = reformer_->convert(vec, ometa, &converted_vector, &converted_meta);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Rabitq hnsw convert failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
level_t level = alg_->get_random_level();
|
||||
node_id_t id;
|
||||
|
||||
if (converted_vector.size() != entity_.vector_size()) {
|
||||
LOG_ERROR(
|
||||
"Converted vector size %zu is not equal to entity vector size %zu",
|
||||
converted_vector.size(), entity_.vector_size());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
ret = entity_.add_vector(level, iter->key(), converted_vector.data(), &id);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
iter->next();
|
||||
}
|
||||
|
||||
LOG_INFO("Finished save vector, start build graph...");
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
std::atomic<node_id_t> finished{0};
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(ailego::Closure ::New(this, &HnswRabitqBuilder::do_build,
|
||||
i, threads->count(), provider,
|
||||
&finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to build index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Built cnt %zu, finished percent %.3f%%",
|
||||
static_cast<size_t>(finished.load()),
|
||||
finished.load() * 100.0f / entity_.doc_cnt());
|
||||
}
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to build index while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
task_group->wait_finish();
|
||||
|
||||
stats_.set_built_count(finished.load());
|
||||
stats_.set_built_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
|
||||
state_ = BUILD_STATE_BUILT;
|
||||
LOG_INFO("End HnswRabitqBuilder::build with RaBitQ quantization");
|
||||
return 0;
|
||||
}
|
||||
|
||||
void HnswRabitqBuilder::do_build(node_id_t idx, size_t step_size,
|
||||
IndexProvider::Pointer provider,
|
||||
std::atomic<node_id_t> *finished) {
|
||||
AILEGO_DEFER([&]() {
|
||||
std::lock_guard<std::mutex> latch(mutex_);
|
||||
cond_.notify_one();
|
||||
});
|
||||
HnswRabitqContext *ctx = new (std::nothrow) HnswRabitqContext(
|
||||
meta_.dimension(), metric_,
|
||||
std::shared_ptr<HnswRabitqEntity>(&entity_, [](HnswRabitqEntity *) {}));
|
||||
if (ailego_unlikely(ctx == nullptr)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("Failed to create context");
|
||||
errcode_ = IndexError_NoMemory;
|
||||
}
|
||||
return;
|
||||
}
|
||||
HnswRabitqContext::Pointer auto_ptr(ctx);
|
||||
ctx->set_provider(std::move(provider));
|
||||
ctx->set_max_scan_num(entity_.doc_cnt());
|
||||
int ret = ctx->init(HnswRabitqContext::kBuilderContext);
|
||||
if (ret != 0) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("Failed to init context");
|
||||
errcode_ = IndexError_Runtime;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (node_id_t id = idx; id < entity_.doc_cnt(); id += step_size) {
|
||||
ctx->reset_query(ctx->dist_calculator().get_vector(id));
|
||||
ret = alg_->add_node(id, entity_.get_level(id), ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
if (!error_.exchange(true)) {
|
||||
LOG_ERROR("Hnsw graph add node failed");
|
||||
errcode_ = ret;
|
||||
}
|
||||
return;
|
||||
}
|
||||
ctx->clear();
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
int HnswRabitqBuilder::dump(const IndexDumper::Pointer &dumper) {
|
||||
if (state_ != BUILD_STATE_BUILT) {
|
||||
LOG_INFO("Build the index before HnswRabitqBuilder::dump");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_INFO("Begin HnswRabitqBuilder::dump");
|
||||
|
||||
meta_.set_searcher("HnswRabitqSearcher", HnswRabitqEntity::kRevision,
|
||||
ailego::Params());
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
int ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to serialize meta into dumper.");
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Dump RaBitQ centroids first
|
||||
if (converter_) {
|
||||
ret = converter_->dump(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to dump RabitqConverter: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
LOG_INFO("RaBitQ centroids dumped: %zu bytes, cost %zu ms",
|
||||
converter_->stats().dumped_size(),
|
||||
static_cast<size_t>(converter_->stats().dumped_costtime()));
|
||||
}
|
||||
|
||||
ret = entity_.dump(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("HnswRabitqBuilder dump index failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
stats_.set_dumped_count(entity_.doc_cnt());
|
||||
stats_.set_dumped_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
|
||||
LOG_INFO("End HnswRabitqBuilder::dump");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include "zvec/core/framework/index_builder.h"
|
||||
#include "zvec/core/framework/index_converter.h"
|
||||
#include "zvec/core/framework/index_reformer.h"
|
||||
#include "hnsw_rabitq_algorithm.h"
|
||||
#include "hnsw_rabitq_builder_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class HnswRabitqBuilder : public IndexBuilder {
|
||||
public:
|
||||
//! Constructor
|
||||
HnswRabitqBuilder();
|
||||
|
||||
//! Initialize the builder
|
||||
virtual int init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup the builder
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(IndexThreads::Pointer,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(const IndexTrainer::Pointer &trainer) override;
|
||||
|
||||
|
||||
//! Build the index
|
||||
virtual int build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Dump index into storage
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
private:
|
||||
void do_build(node_id_t idx, size_t step_size,
|
||||
IndexProvider::Pointer provider,
|
||||
std::atomic<node_id_t> *finished);
|
||||
|
||||
int train_converter_and_load_reformer(IndexHolder::Pointer holder);
|
||||
|
||||
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
|
||||
constexpr static uint32_t kMaxNeighborCnt = 65535;
|
||||
|
||||
private:
|
||||
enum BUILD_STATE {
|
||||
BUILD_STATE_INIT = 0,
|
||||
BUILD_STATE_INITED = 1,
|
||||
BUILD_STATE_TRAINED = 2,
|
||||
BUILD_STATE_BUILT = 3
|
||||
};
|
||||
|
||||
HnswRabitqBuilderEntity entity_{};
|
||||
HnswRabitqAlgorithm::UPointer alg_; // impl graph algorithm
|
||||
uint32_t thread_cnt_{0};
|
||||
uint32_t min_neighbor_cnt_{0};
|
||||
uint32_t upper_max_neighbor_cnt_{
|
||||
HnswRabitqEntity::kDefaultUpperMaxNeighborCnt};
|
||||
uint32_t l0_max_neighbor_cnt_{HnswRabitqEntity::kDefaultL0MaxNeighborCnt};
|
||||
uint32_t ef_construction_{HnswRabitqEntity::kDefaultEfConstruction};
|
||||
uint32_t scaling_factor_{HnswRabitqEntity::kDefaultScalingFactor};
|
||||
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
|
||||
|
||||
int errcode_{0};
|
||||
std::atomic_bool error_{false};
|
||||
IndexMeta meta_{};
|
||||
IndexMetric::Pointer metric_{};
|
||||
IndexConverter::Pointer converter_{}; // RaBitQ converter
|
||||
IndexReformer::Pointer reformer_{}; // RaBitQ reformer
|
||||
std::mutex mutex_{};
|
||||
std::condition_variable cond_{};
|
||||
Stats stats_{};
|
||||
|
||||
BUILD_STATE state_{BUILD_STATE_INIT};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,199 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_builder_entity.h"
|
||||
#include <iostream>
|
||||
#include <zvec/ailego/hash/crc32c.h>
|
||||
#include "utility/sparse_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswRabitqBuilderEntity::HnswRabitqBuilderEntity() {
|
||||
update_ep_and_level(kInvalidNodeId, 0U);
|
||||
}
|
||||
|
||||
int HnswRabitqBuilderEntity::cleanup() {
|
||||
memory_quota_ = 0UL;
|
||||
neighbors_size_ = 0U;
|
||||
upper_neighbors_size_ = 0U;
|
||||
padding_size_ = 0U;
|
||||
vectors_buffer_.clear();
|
||||
keys_buffer_.clear();
|
||||
neighbors_buffer_.clear();
|
||||
upper_neighbors_buffer_.clear();
|
||||
neighbors_index_.clear();
|
||||
|
||||
vectors_buffer_.shrink_to_fit();
|
||||
keys_buffer_.shrink_to_fit();
|
||||
neighbors_buffer_.shrink_to_fit();
|
||||
upper_neighbors_buffer_.shrink_to_fit();
|
||||
neighbors_index_.shrink_to_fit();
|
||||
|
||||
this->HnswRabitqEntity::cleanup();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilderEntity::init() {
|
||||
size_t size = vector_size();
|
||||
|
||||
//! aligned size to 32
|
||||
set_node_size(AlignSize(size));
|
||||
//! if node size is aligned to 1k, the build performance will downgrade
|
||||
if (node_size() % 1024 == 0) {
|
||||
set_node_size(AlignSize(node_size() + 1));
|
||||
}
|
||||
|
||||
padding_size_ = node_size() - size;
|
||||
|
||||
neighbors_size_ = neighbors_size();
|
||||
upper_neighbors_size_ = upper_neighbors_size();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilderEntity::reserve_space(size_t docs) {
|
||||
if (memory_quota_ > 0 && (node_size() * docs + neighbors_size_ * docs +
|
||||
sizeof(NeighborIndex) * docs >
|
||||
memory_quota_)) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
|
||||
vectors_buffer_.reserve(node_size() * docs);
|
||||
keys_buffer_.reserve(sizeof(key_t) * docs);
|
||||
neighbors_buffer_.reserve(neighbors_size_ * docs);
|
||||
neighbors_index_.reserve(docs);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilderEntity::add_vector(level_t level, key_t key,
|
||||
const void *vec, node_id_t *id) {
|
||||
if (memory_quota_ > 0 &&
|
||||
(vectors_buffer_.capacity() + keys_buffer_.capacity() +
|
||||
neighbors_buffer_.capacity() + upper_neighbors_buffer_.capacity() +
|
||||
neighbors_index_.capacity() * sizeof(NeighborIndex)) > memory_quota_) {
|
||||
LOG_ERROR("Add vector failed, used memory exceed quota, cur_doc=%zu",
|
||||
static_cast<size_t>(doc_cnt()));
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
|
||||
vectors_buffer_.append(reinterpret_cast<const char *>(vec), vector_size());
|
||||
vectors_buffer_.append(padding_size_, '\0');
|
||||
keys_buffer_.append(reinterpret_cast<const char *>(&key), sizeof(key));
|
||||
|
||||
// init level 0 neighbors
|
||||
neighbors_buffer_.append(neighbors_size_, '\0');
|
||||
|
||||
neighbors_index_.emplace_back(upper_neighbors_buffer_.size(), level);
|
||||
|
||||
// init upper layer neighbors
|
||||
for (level_t cur_level = 1; cur_level <= level; ++cur_level) {
|
||||
upper_neighbors_buffer_.append(upper_neighbors_size_, '\0');
|
||||
}
|
||||
|
||||
*id = (*mutable_doc_cnt())++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
key_t HnswRabitqBuilderEntity::get_key(node_id_t id) const {
|
||||
return *(reinterpret_cast<const key_t *>(keys_buffer_.data() +
|
||||
id * sizeof(key_t)));
|
||||
}
|
||||
|
||||
const void *HnswRabitqBuilderEntity::get_vector(node_id_t id) const {
|
||||
return vectors_buffer_.data() + id * node_size();
|
||||
}
|
||||
|
||||
int HnswRabitqBuilderEntity::get_vector(
|
||||
const node_id_t id, IndexStorage::MemoryBlock &block) const {
|
||||
const void *vec = get_vector(id);
|
||||
block.reset((void *)vec);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilderEntity::get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const {
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
vecs[i] = vectors_buffer_.data() + ids[i] * node_size();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilderEntity::get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
|
||||
std::vector<const void *> vecs(count);
|
||||
get_vector(ids, count, vecs.data());
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i]));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const Neighbors HnswRabitqBuilderEntity::get_neighbors(level_t level,
|
||||
node_id_t id) const {
|
||||
const NeighborsHeader *hd = get_neighbor_header(level, id);
|
||||
return {hd->neighbor_cnt, hd->neighbors};
|
||||
}
|
||||
|
||||
int HnswRabitqBuilderEntity::update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors) {
|
||||
NeighborsHeader *hd =
|
||||
const_cast<NeighborsHeader *>(get_neighbor_header(level, id));
|
||||
for (size_t i = 0; i < neighbors.size(); ++i) {
|
||||
hd->neighbors[i] = neighbors[i].first;
|
||||
}
|
||||
hd->neighbor_cnt = neighbors.size();
|
||||
|
||||
// std::cout << "id: " << id << ", neighbour, id: ";
|
||||
// for (size_t i = 0; i < neighbors.size(); ++i) {
|
||||
// if (i == neighbors.size()-1)
|
||||
// std::cout << neighbors[i].first << ", score:" << neighbors[i].second <<
|
||||
// std::endl;
|
||||
// else
|
||||
// std::cout << neighbors[i].first << ", score:" << neighbors[i].second <<
|
||||
// ", id: ";
|
||||
// }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void HnswRabitqBuilderEntity::add_neighbor(level_t level, node_id_t id,
|
||||
uint32_t /*size*/,
|
||||
node_id_t neighbor_id) {
|
||||
NeighborsHeader *hd =
|
||||
const_cast<NeighborsHeader *>(get_neighbor_header(level, id));
|
||||
hd->neighbors[hd->neighbor_cnt++] = neighbor_id;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
int HnswRabitqBuilderEntity::dump(const IndexDumper::Pointer &dumper) {
|
||||
key_t *keys =
|
||||
reinterpret_cast<key_t *>(const_cast<char *>(keys_buffer_.data()));
|
||||
auto ret =
|
||||
dump_segments(dumper, keys, [&](node_id_t id) { return get_level(id); });
|
||||
if (ailego_unlikely(ret < 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,139 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class HnswRabitqBuilderEntity : public HnswRabitqEntity {
|
||||
public:
|
||||
//! Add vector and key to hnsw entity, and local id will be saved to id
|
||||
virtual int add_vector(level_t level, key_t key, const void *vec,
|
||||
node_id_t *id) override;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector(node_id_t id) const override;
|
||||
|
||||
//! Batch get vectors feature data by keys
|
||||
virtual int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
|
||||
virtual int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
virtual int get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
const NeighborsHeader *get_neighbor_header(level_t level,
|
||||
node_id_t id) const {
|
||||
if (level == 0) {
|
||||
return reinterpret_cast<const NeighborsHeader *>(
|
||||
neighbors_buffer_.data() + neighbors_size_ * id);
|
||||
} else {
|
||||
size_t offset = neighbors_index_[id].offset;
|
||||
return reinterpret_cast<const NeighborsHeader *>(
|
||||
upper_neighbors_buffer_.data() + offset +
|
||||
(level - 1) * upper_neighbors_size_);
|
||||
}
|
||||
}
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
virtual const Neighbors get_neighbors(level_t level,
|
||||
node_id_t id) const override;
|
||||
|
||||
//! Replace node id in level's neighbors
|
||||
virtual int update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors)
|
||||
override;
|
||||
|
||||
//! add a neighbor to id in graph level
|
||||
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
|
||||
//! Dump the hnsw graph to dumper
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Cleanup the entity
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
public:
|
||||
//! Constructor
|
||||
HnswRabitqBuilderEntity();
|
||||
|
||||
//! Get the node graph level by id
|
||||
level_t get_level(node_id_t id) const {
|
||||
return neighbors_index_[id].level;
|
||||
}
|
||||
|
||||
//! Init builerEntity
|
||||
int init();
|
||||
|
||||
//! reserve buffer space for documents
|
||||
//! @param docs number of documents
|
||||
int reserve_space(size_t docs);
|
||||
|
||||
//! Set memory quota params
|
||||
inline void set_memory_quota(size_t memory_quota) {
|
||||
memory_quota_ = memory_quota;
|
||||
}
|
||||
|
||||
//! Get neighbors size
|
||||
inline size_t neighbors_size() const {
|
||||
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
|
||||
}
|
||||
|
||||
//! Get upper neighbors size
|
||||
inline size_t upper_neighbors_size() const {
|
||||
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
|
||||
}
|
||||
|
||||
public:
|
||||
HnswRabitqBuilderEntity(const HnswRabitqBuilderEntity &) = delete;
|
||||
HnswRabitqBuilderEntity &operator=(const HnswRabitqBuilderEntity &) = delete;
|
||||
|
||||
private:
|
||||
friend class HnswRabitqSearcherEntity;
|
||||
//! class internal used only
|
||||
struct NeighborIndex {
|
||||
NeighborIndex(size_t off, level_t l) : offset(off), level(l) {}
|
||||
uint64_t offset : 48;
|
||||
uint64_t level : 16;
|
||||
};
|
||||
|
||||
std::string vectors_buffer_{}; // aligned vectors
|
||||
std::string keys_buffer_{}; // aligned vectors
|
||||
std::string neighbors_buffer_{}; // level 0 neighbors buffer
|
||||
std::string upper_neighbors_buffer_{}; // upper layer neighbors buffer
|
||||
|
||||
std::string sparse_data_buffer_{}; // aligned spase data buffer
|
||||
size_t sparse_data_offset_{0}; //
|
||||
|
||||
// upper layer offset + level in upper_neighbors_buffer_
|
||||
std::vector<NeighborIndex> neighbors_index_{};
|
||||
size_t memory_quota_{0UL};
|
||||
size_t neighbors_size_{0U}; // level 0 neighbors size
|
||||
size_t upper_neighbors_size_{0U}; // level 0 neighbors size
|
||||
size_t padding_size_{}; // padding size for each vector element
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,221 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_chunk.h"
|
||||
#include <chrono>
|
||||
#include <random>
|
||||
#include <zvec/ailego/hash/crc32c.h>
|
||||
#include <zvec/ailego/utility/time_helper.h>
|
||||
#include "zvec/core/framework/index_error.h"
|
||||
#include "zvec/core/framework/index_helper.h"
|
||||
#include "zvec/core/framework/index_logger.h"
|
||||
#include "zvec/core/framework/index_streamer.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int HnswRabitqChunkBroker::init_storage(size_t chunk_size) {
|
||||
chunk_meta_.clear();
|
||||
chunk_meta_.chunk_size = chunk_size;
|
||||
chunk_meta_.create_time = ailego::Realtime::Seconds();
|
||||
stats_.set_create_time(chunk_meta_.create_time);
|
||||
chunk_meta_.update_time = ailego::Realtime::Seconds();
|
||||
stats_.set_update_time(chunk_meta_.update_time);
|
||||
|
||||
//! alloc meta chunk
|
||||
size_t size = sizeof(HnswChunkMeta);
|
||||
size = (size + page_mask_) & (~page_mask_);
|
||||
const std::string segment_id =
|
||||
make_segment_id(CHUNK_TYPE_META, kDefaultChunkSeqId);
|
||||
int ret = stg_->append(segment_id, size);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Storage append segment failed for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
chunk_meta_segment_ = get_chunk(CHUNK_TYPE_META, kDefaultChunkSeqId);
|
||||
if (ailego_unlikely(!chunk_meta_segment_)) {
|
||||
LOG_ERROR("Get meta segment failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
//! update meta info and write to storage
|
||||
chunk_meta_.chunk_cnts[CHUNK_TYPE_META] += 1;
|
||||
chunk_meta_.total_size += size;
|
||||
(*stats_.mutable_index_size()) += size;
|
||||
size = chunk_meta_segment_->write(0UL, &chunk_meta_, sizeof(HnswChunkMeta));
|
||||
if (ailego_unlikely(size != sizeof(HnswChunkMeta))) {
|
||||
LOG_ERROR("Storage write data failed, wsize=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqChunkBroker::load_storage(size_t chunk_size) {
|
||||
IndexStorage::MemoryBlock data_block;
|
||||
size_t size = chunk_meta_segment_->read(0UL, data_block,
|
||||
chunk_meta_segment_->data_size());
|
||||
if (size != sizeof(HnswChunkMeta)) {
|
||||
LOG_ERROR("Invalid hnsw meta chunk, read size=%zu chunk size=%zu", size,
|
||||
chunk_meta_segment_->data_size());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
std::memcpy(&chunk_meta_, data_block.data(), size);
|
||||
if (chunk_meta_.chunk_size != chunk_size) {
|
||||
LOG_ERROR(
|
||||
"Params hnsw chunk size=%zu mismatch from previous %zu "
|
||||
"in index",
|
||||
chunk_size, (size_t)chunk_meta_.chunk_size);
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
*stats_.mutable_check_point() = stg_->check_point();
|
||||
stats_.set_revision_id(chunk_meta_.revision_id);
|
||||
stats_.set_update_time(chunk_meta_.update_time);
|
||||
stats_.set_create_time(chunk_meta_.create_time);
|
||||
|
||||
char create_time[32];
|
||||
char update_time[32];
|
||||
ailego::Realtime::Gmtime(chunk_meta_.create_time, "%Y-%m-%d %H:%M:%S",
|
||||
create_time, sizeof(create_time));
|
||||
ailego::Realtime::Gmtime(chunk_meta_.update_time, "%Y-%m-%d %H:%M:%S",
|
||||
update_time, sizeof(update_time));
|
||||
LOG_DEBUG(
|
||||
"Load index, indexSize=%zu chunkSize=%zu nodeChunks=%zu "
|
||||
"upperNeighborChunks=%zu revisionId=%zu "
|
||||
"createTime=%s updateTime=%s",
|
||||
(size_t)chunk_meta_.total_size, (size_t)chunk_meta_.chunk_size,
|
||||
(size_t)chunk_meta_.chunk_cnts[CHUNK_TYPE_NODE],
|
||||
(size_t)chunk_meta_.chunk_cnts[CHUNK_TYPE_UPPER_NEIGHBOR],
|
||||
(size_t)chunk_meta_.revision_id, create_time, update_time);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqChunkBroker::open(IndexStorage::Pointer stg,
|
||||
size_t max_index_size, size_t chunk_size,
|
||||
bool check_crc) {
|
||||
if (ailego_unlikely(stg_)) {
|
||||
LOG_ERROR("An storage instance is already opened");
|
||||
return IndexError_Duplicate;
|
||||
}
|
||||
stg_ = std::move(stg);
|
||||
if (stg_->isHugePage()) {
|
||||
page_mask_ = ailego::MemoryHelper::HugePageSize() - 1;
|
||||
} else {
|
||||
page_mask_ = ailego::MemoryHelper::PageSize() - 1;
|
||||
}
|
||||
check_crc_ = check_crc;
|
||||
max_chunks_size_ = max_index_size;
|
||||
dirty_ = false;
|
||||
|
||||
const std::string segment_id =
|
||||
make_segment_id(CHUNK_TYPE_META, kDefaultChunkSeqId);
|
||||
chunk_meta_segment_ = stg_->get(segment_id);
|
||||
if (!chunk_meta_segment_) {
|
||||
LOG_DEBUG("Create new index");
|
||||
return init_storage(chunk_size);
|
||||
}
|
||||
|
||||
return load_storage(chunk_size);
|
||||
}
|
||||
|
||||
int HnswRabitqChunkBroker::close(void) {
|
||||
flush(0UL);
|
||||
|
||||
stg_.reset();
|
||||
check_crc_ = false;
|
||||
dirty_ = false;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqChunkBroker::flush(uint64_t checkpoint) {
|
||||
ailego_assert_with(chunk_meta_segment_, "invalid meta segment");
|
||||
|
||||
chunk_meta_.update_time = ailego::Realtime::Seconds();
|
||||
stats_.set_update_time(chunk_meta_.update_time);
|
||||
|
||||
size_t size =
|
||||
chunk_meta_segment_->write(0UL, &chunk_meta_, sizeof(HnswChunkMeta));
|
||||
if (ailego_unlikely(size != sizeof(HnswChunkMeta))) {
|
||||
LOG_ERROR("Storage write data failed, wsize=%zu", size);
|
||||
}
|
||||
|
||||
stg_->refresh(checkpoint);
|
||||
int ret = stg_->flush();
|
||||
if (ret == 0) {
|
||||
(*stats_.mutable_check_point()) = checkpoint;
|
||||
} else {
|
||||
LOG_ERROR("Storage flush failed for %s", IndexError::What(ret));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::pair<int, Chunk::Pointer> HnswRabitqChunkBroker::alloc_chunk(
|
||||
int type, uint64_t seq_id, size_t size) {
|
||||
ailego_assert_with(type < CHUNK_TYPE_MAX, "chunk type overflow");
|
||||
|
||||
Chunk::Pointer chunk;
|
||||
if (ailego_unlikely(!stg_)) {
|
||||
LOG_ERROR("Init storage first");
|
||||
return std::make_pair(IndexError_Uninitialized, chunk);
|
||||
}
|
||||
|
||||
//! check exist a empty chunk with the same name
|
||||
chunk = get_chunk(type, seq_id);
|
||||
if (chunk) {
|
||||
if (ailego_unlikely(chunk->capacity() == size &&
|
||||
chunk->data_size() == 0UL)) {
|
||||
LOG_ERROR("Exist invalid chunk size %zu, expect size %zu",
|
||||
chunk->capacity(), size);
|
||||
chunk.reset();
|
||||
return std::make_pair(IndexError_Runtime, chunk);
|
||||
}
|
||||
return std::make_pair(0, chunk);
|
||||
}
|
||||
//! align to page size
|
||||
size = (size + page_mask_) & (~page_mask_);
|
||||
if (ailego_unlikely(chunk_meta_.total_size + size >= max_chunks_size_)) {
|
||||
LOG_ERROR("No space to new a chunk, curIndexSize=%zu allocSize=%zu",
|
||||
(size_t)chunk_meta_.total_size, size);
|
||||
return std::make_pair(IndexError_IndexFull, chunk);
|
||||
}
|
||||
|
||||
std::string segment_id = make_segment_id(type, seq_id);
|
||||
int ret = stg_->append(segment_id, size);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Storage append segment failed for %s", IndexError::What(ret));
|
||||
return std::make_pair(ret, chunk);
|
||||
}
|
||||
chunk_meta_.chunk_cnts[type] += 1;
|
||||
chunk_meta_.total_size += size;
|
||||
(*stats_.mutable_index_size()) += size;
|
||||
|
||||
size = chunk_meta_segment_->write(0UL, &chunk_meta_, sizeof(HnswChunkMeta));
|
||||
if (ailego_unlikely(size != sizeof(HnswChunkMeta))) {
|
||||
LOG_ERROR("Storage append segment failed, wsize=%zu", size);
|
||||
}
|
||||
|
||||
chunk = get_chunk(type, seq_id);
|
||||
return std::make_pair(chunk ? 0 : IndexError_NoMemory, chunk);
|
||||
}
|
||||
|
||||
Chunk::Pointer HnswRabitqChunkBroker::get_chunk(int type,
|
||||
uint64_t seq_id) const {
|
||||
std::string segment_id = make_segment_id(type, seq_id);
|
||||
return stg_->get(segment_id);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,140 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <atomic>
|
||||
#include <cstddef>
|
||||
#include <mutex>
|
||||
#include <ailego/utility/memory_helper.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include <zvec/ailego/utility/string_helper.h>
|
||||
#include "zvec/core/framework/index_error.h"
|
||||
#include "zvec/core/framework/index_logger.h"
|
||||
#include "zvec/core/framework/index_storage.h"
|
||||
#include "zvec/core/framework/index_streamer.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
using Chunk = IndexStorage::Segment;
|
||||
|
||||
class HnswRabitqChunkBroker {
|
||||
public:
|
||||
typedef std::shared_ptr<HnswRabitqChunkBroker> Pointer;
|
||||
|
||||
enum CHUNK_TYPE {
|
||||
CHUNK_TYPE_HEADER = 1,
|
||||
CHUNK_TYPE_META = 2,
|
||||
CHUNK_TYPE_NODE = 3,
|
||||
CHUNK_TYPE_UPPER_NEIGHBOR = 4,
|
||||
CHUNK_TYPE_NEIGHBOR_INDEX = 5,
|
||||
CHUNK_TYPE_SPARSE_NODE = 6,
|
||||
CHUNK_TYPE_MAX = 8
|
||||
};
|
||||
static constexpr size_t kDefaultChunkSeqId = 0UL;
|
||||
|
||||
HnswRabitqChunkBroker(IndexStreamer::Stats &stats) : stats_(stats) {}
|
||||
|
||||
//! Open storage
|
||||
int open(IndexStorage::Pointer stg, size_t max_index_size, size_t chunk_size,
|
||||
bool check_crc);
|
||||
|
||||
int close(void);
|
||||
|
||||
int flush(uint64_t checkpoint);
|
||||
|
||||
//! alloc a new chunk with size, not thread-safe
|
||||
std::pair<int, Chunk::Pointer> alloc_chunk(int type, uint64_t seq_id,
|
||||
size_t size);
|
||||
|
||||
//! alloc a new chunk with chunk size
|
||||
inline std::pair<int, Chunk::Pointer> alloc_chunk(int type, uint64_t seq_id) {
|
||||
return alloc_chunk(type, seq_id, chunk_meta_.chunk_size);
|
||||
}
|
||||
|
||||
Chunk::Pointer get_chunk(int type, uint64_t seq_id) const;
|
||||
|
||||
inline size_t get_chunk_cnt(int type) const {
|
||||
ailego_assert_with(type < CHUNK_TYPE_MAX, "chunk type overflow");
|
||||
return chunk_meta_.chunk_cnts[type];
|
||||
}
|
||||
|
||||
inline bool dirty(void) const {
|
||||
return dirty_;
|
||||
}
|
||||
|
||||
inline void mark_dirty(void) {
|
||||
if (!dirty_) {
|
||||
dirty_ = true;
|
||||
chunk_meta_.revision_id += 1;
|
||||
stats_.set_revision_id(chunk_meta_.revision_id);
|
||||
}
|
||||
}
|
||||
|
||||
const IndexStorage::Pointer storage(void) const {
|
||||
return stg_;
|
||||
}
|
||||
|
||||
private:
|
||||
HnswRabitqChunkBroker(const HnswRabitqChunkBroker &) = delete;
|
||||
HnswRabitqChunkBroker &operator=(const HnswRabitqChunkBroker &) = delete;
|
||||
|
||||
struct HnswChunkMeta {
|
||||
HnswChunkMeta(void) {
|
||||
memset(this, 0, sizeof(HnswChunkMeta));
|
||||
}
|
||||
void clear() {
|
||||
memset(this, 0, sizeof(HnswChunkMeta));
|
||||
}
|
||||
|
||||
uint64_t chunk_cnts[CHUNK_TYPE_MAX];
|
||||
uint64_t chunk_size; // size of per chunk
|
||||
uint64_t total_size; // total size of allocated chunk
|
||||
uint64_t revision_id; // index revision
|
||||
uint64_t create_time;
|
||||
uint64_t update_time;
|
||||
uint64_t reserved[3];
|
||||
};
|
||||
|
||||
static_assert(sizeof(HnswChunkMeta) % 32 == 0,
|
||||
"HnswChunkMeta must be aligned with 32 bytes");
|
||||
|
||||
//! Init the storage after open an empty index
|
||||
int init_storage(size_t chunk_size);
|
||||
|
||||
//! Load index from storage
|
||||
int load_storage(size_t chunk_size);
|
||||
|
||||
static inline const std::string make_segment_id(int type, uint64_t seq_id) {
|
||||
return "HnswT" + ailego::StringHelper::ToString(type) + "S" +
|
||||
ailego::StringHelper::ToString(seq_id);
|
||||
}
|
||||
|
||||
private:
|
||||
IndexStreamer::Stats &stats_;
|
||||
HnswChunkMeta chunk_meta_{};
|
||||
size_t page_mask_{0UL};
|
||||
size_t max_chunks_size_{0UL};
|
||||
IndexStorage::Pointer stg_{};
|
||||
IndexStorage::Segment::Pointer chunk_meta_segment_{};
|
||||
bool check_crc_{false};
|
||||
bool dirty_{false}; // set as true if index is modified , the flag
|
||||
// will not be cleared even if flushed
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,297 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_context.h"
|
||||
#include <chrono>
|
||||
#include "hnsw_rabitq_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswRabitqContext::HnswRabitqContext(size_t dimension,
|
||||
const IndexMetric::Pointer &metric,
|
||||
const HnswRabitqEntity::Pointer &entity)
|
||||
: IndexContext(metric),
|
||||
entity_(entity),
|
||||
add_dc_(entity_.get(), metric, dimension) {}
|
||||
|
||||
HnswRabitqContext::HnswRabitqContext(const IndexMetric::Pointer &metric,
|
||||
const HnswRabitqEntity::Pointer &entity)
|
||||
: IndexContext(metric), entity_(entity), add_dc_(entity_.get(), metric) {}
|
||||
|
||||
HnswRabitqContext::~HnswRabitqContext() {
|
||||
visit_filter_.destroy();
|
||||
}
|
||||
|
||||
int HnswRabitqContext::init(ContextType type) {
|
||||
int ret;
|
||||
uint32_t doc_cnt;
|
||||
|
||||
type_ = type;
|
||||
|
||||
switch (type) {
|
||||
case kBuilderContext:
|
||||
ret = visit_filter_.init(VisitFilter::ByteMap, entity_->doc_cnt(),
|
||||
max_scan_num_, negative_probability_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
candidates_.limit(max_scan_num_);
|
||||
update_heap_.limit(entity_->l0_neighbor_cnt() + 1);
|
||||
break;
|
||||
|
||||
case kSearcherContext:
|
||||
ret = visit_filter_.init(filter_mode_, entity_->doc_cnt(), max_scan_num_,
|
||||
negative_probability_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
candidates_.limit(max_scan_num_);
|
||||
break;
|
||||
|
||||
case kStreamerContext:
|
||||
// maxScanNum is unknown if inited from streamer, so the docCnt may
|
||||
// change. we need to compute maxScanNum by scan ratio, and preserve
|
||||
// max_doc_cnt space from visit filter
|
||||
doc_cnt = entity_->doc_cnt();
|
||||
max_scan_num_ = compute_max_scan_num(doc_cnt);
|
||||
reserve_max_doc_cnt_ = doc_cnt + compute_reserve_cnt(doc_cnt);
|
||||
ret = visit_filter_.init(filter_mode_, reserve_max_doc_cnt_,
|
||||
max_scan_num_, negative_probability_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
|
||||
update_heap_.limit(entity_->l0_neighbor_cnt() + 1);
|
||||
candidates_.limit(max_scan_num_);
|
||||
|
||||
check_need_adjuct_ctx();
|
||||
break;
|
||||
|
||||
default:
|
||||
LOG_ERROR("Init context failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqContext::update(const ailego::Params ¶ms) {
|
||||
auto update_visit_filter_param = [&]() {
|
||||
bool need_update = false;
|
||||
std::string p;
|
||||
switch (type_) {
|
||||
case kSearcherContext:
|
||||
p = PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_ENABLE;
|
||||
break;
|
||||
case kStreamerContext:
|
||||
p = PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_ENABLE;
|
||||
break;
|
||||
}
|
||||
|
||||
if (params.has(p)) {
|
||||
bool bf_enabled;
|
||||
params.get(p, &bf_enabled);
|
||||
if (bf_enabled ^ (filter_mode_ == VisitFilter::BloomFilter)) {
|
||||
need_update = true;
|
||||
filter_mode_ =
|
||||
bf_enabled ? VisitFilter::BloomFilter : VisitFilter::ByteMap;
|
||||
}
|
||||
}
|
||||
|
||||
float prob = negative_probability_;
|
||||
p.clear();
|
||||
switch (type_) {
|
||||
case kSearcherContext:
|
||||
p = PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB;
|
||||
break;
|
||||
case kStreamerContext:
|
||||
p = PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB;
|
||||
break;
|
||||
}
|
||||
params.get(p, &prob);
|
||||
if (filter_mode_ == VisitFilter::BloomFilter &&
|
||||
std::abs(prob - negative_probability_) > 1e-6) {
|
||||
need_update = true;
|
||||
}
|
||||
if (need_update) {
|
||||
visit_filter_.destroy();
|
||||
int max_doc_cnt = 0;
|
||||
if (type_ == kSearcherContext) {
|
||||
max_doc_cnt = entity_->doc_cnt();
|
||||
} else {
|
||||
max_doc_cnt = reserve_max_doc_cnt_;
|
||||
}
|
||||
int ret = visit_filter_.init(filter_mode_, max_doc_cnt, max_scan_num_,
|
||||
negative_probability_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
|
||||
switch (type_) {
|
||||
case kSearcherContext:
|
||||
if (params.has(PARAM_HNSW_RABITQ_SEARCHER_EF)) {
|
||||
params.get(PARAM_HNSW_RABITQ_SEARCHER_EF, &ef_);
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
}
|
||||
|
||||
if (params.has(PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO)) {
|
||||
params.get(PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO, &max_scan_ratio_);
|
||||
max_scan_num_ =
|
||||
static_cast<uint32_t>(max_scan_ratio_ * entity_->doc_cnt());
|
||||
max_scan_num_ = std::max(10000U, max_scan_num_);
|
||||
}
|
||||
|
||||
if (params.has(PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD)) {
|
||||
params.get(PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD,
|
||||
&bruteforce_threshold_);
|
||||
}
|
||||
|
||||
return update_visit_filter_param();
|
||||
|
||||
case kStreamerContext:
|
||||
if (params.has(PARAM_HNSW_RABITQ_STREAMER_EF)) {
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_EF, &ef_);
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
}
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_EF, &ef_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
|
||||
if (max_scan_ratio_ <= 0.0f || max_scan_ratio_ > 1.0f) {
|
||||
LOG_ERROR("[%s] must be in range (0.0f,1.0f]",
|
||||
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (max_scan_limit_ < min_scan_limit_) {
|
||||
LOG_ERROR("[%s] must be >= [%s]",
|
||||
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT.c_str(),
|
||||
PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (params.has(PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD)) {
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD,
|
||||
&bruteforce_threshold_);
|
||||
}
|
||||
|
||||
return update_visit_filter_param();
|
||||
|
||||
default:
|
||||
LOG_ERROR("update context failed, type=%zu", static_cast<size_t>(type_));
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
|
||||
int HnswRabitqContext::update_context(ContextType type, const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &metric,
|
||||
const HnswRabitqEntity::Pointer &entity,
|
||||
uint32_t magic_num) {
|
||||
uint32_t doc_cnt;
|
||||
|
||||
if (ailego_unlikely(type != type_)) {
|
||||
LOG_ERROR(
|
||||
"HnswRabitqContext doesn't support shared by different type, "
|
||||
"src=%u dst=%u",
|
||||
type_, type);
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
magic_ = kInvalidMgic;
|
||||
|
||||
// TODO: support change filter mode?
|
||||
switch (type) {
|
||||
case kBuilderContext:
|
||||
LOG_ERROR("BuildContext doesn't support update");
|
||||
return IndexError_NotImplemented;
|
||||
|
||||
case kSearcherContext:
|
||||
if (!visit_filter_.reset(entity->doc_cnt(), max_scan_num_)) {
|
||||
LOG_ERROR("Reset filter failed, mode %d", visit_filter_.get_mode());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
candidates_.limit(max_scan_num_);
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
break;
|
||||
|
||||
case kStreamerContext:
|
||||
doc_cnt = entity->doc_cnt();
|
||||
max_scan_num_ = compute_max_scan_num(doc_cnt);
|
||||
reserve_max_doc_cnt_ = doc_cnt + compute_reserve_cnt(doc_cnt);
|
||||
if (!visit_filter_.reset(reserve_max_doc_cnt_, max_scan_num_)) {
|
||||
LOG_ERROR("Reset filter failed, mode %d", visit_filter_.get_mode());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
update_heap_.limit(entity->l0_neighbor_cnt() + 1);
|
||||
candidates_.limit(max_scan_num_);
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
break;
|
||||
|
||||
default:
|
||||
LOG_ERROR("update context failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
entity_ = entity;
|
||||
dc().update(entity_.get(), metric, meta.dimension());
|
||||
magic_ = magic_num;
|
||||
level_topks_.clear();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void HnswRabitqContext::fill_random_to_topk_full(void) {
|
||||
static std::mt19937 mt(
|
||||
std::chrono::system_clock::now().time_since_epoch().count());
|
||||
std::uniform_int_distribution<node_id_t> dt(0, entity_->doc_cnt() - 1);
|
||||
std::function<node_id_t()> gen;
|
||||
node_id_t seqid;
|
||||
std::function<bool(node_id_t)> myfilter = [](node_id_t) { return false; };
|
||||
if (this->filter().is_valid()) {
|
||||
myfilter = [&](node_id_t id) {
|
||||
return this->filter()(entity_->get_key(id));
|
||||
};
|
||||
}
|
||||
|
||||
if (topk_heap_.limit() < entity_->doc_cnt() / 2) {
|
||||
gen = [&](void) { return dt(mt); };
|
||||
} else {
|
||||
// If topk limit is big value, gen sequential id from an random initial
|
||||
seqid = dt(mt);
|
||||
gen = [&](void) {
|
||||
seqid = seqid == (entity_->doc_cnt() - 1) ? 0 : (seqid + 1);
|
||||
return seqid;
|
||||
};
|
||||
}
|
||||
|
||||
for (size_t i = 0; !topk_heap_.full() && i < entity_->doc_cnt(); ++i) {
|
||||
const auto id = gen();
|
||||
if (!visit_filter_.visited(id) && !myfilter(id)) {
|
||||
visit_filter_.set_visited(id);
|
||||
topk_heap_.emplace(id, dc().dist(id));
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,541 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/ailego/logger/logger.h>
|
||||
#include "utility/visit_filter.h"
|
||||
#include "zvec/core/framework/index_context.h"
|
||||
#include "zvec/core/framework/index_provider.h"
|
||||
#include "hnsw_rabitq_dist_calculator.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class HnswRabitqContext : public IndexContext {
|
||||
public:
|
||||
//! Index Context Pointer
|
||||
typedef std::unique_ptr<HnswRabitqContext> Pointer;
|
||||
|
||||
enum ContextType {
|
||||
kUnknownContext = 0,
|
||||
kSearcherContext = 1,
|
||||
kBuilderContext = 2,
|
||||
kStreamerContext = 3
|
||||
};
|
||||
|
||||
//! Construct
|
||||
HnswRabitqContext(size_t dimension, const IndexMetric::Pointer &metric,
|
||||
const HnswRabitqEntity::Pointer &entity);
|
||||
|
||||
//! Construct
|
||||
HnswRabitqContext(const IndexMetric::Pointer &metric,
|
||||
const HnswRabitqEntity::Pointer &entity);
|
||||
|
||||
//! Destructor
|
||||
virtual ~HnswRabitqContext();
|
||||
|
||||
public:
|
||||
//! Set topk of search result
|
||||
virtual void set_topk(uint32_t val) override {
|
||||
topk_ = val;
|
||||
topk_heap_.limit(std::max(val, ef_));
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve result object for output
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(void) const override {
|
||||
return group_results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(
|
||||
size_t idx) const override {
|
||||
return group_results_[idx];
|
||||
}
|
||||
|
||||
virtual uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
//! Set mode of debug
|
||||
virtual void set_debug_mode(bool enable) override {
|
||||
debug_mode_ = enable;
|
||||
}
|
||||
|
||||
//! Retrieve mode of debug
|
||||
virtual bool debug_mode(void) const override {
|
||||
return this->debugging();
|
||||
}
|
||||
|
||||
//! Retrieve string of debug
|
||||
virtual std::string debug_string(void) const override {
|
||||
char buf[4096];
|
||||
size_t size = snprintf(
|
||||
buf, sizeof(buf),
|
||||
"scan_cnt=%zu,get_vector_cnt=%u,get_neighbors_cnt=%u,dup_node=%u",
|
||||
get_scan_num(), stats_get_vector_cnt_, stats_get_neighbors_cnt_,
|
||||
stats_visit_dup_cnt_);
|
||||
return std::string(buf, size);
|
||||
}
|
||||
|
||||
//! Update the parameters of context
|
||||
virtual int update(const ailego::Params ¶ms) override;
|
||||
|
||||
public:
|
||||
//! Init context
|
||||
int init(ContextType type);
|
||||
|
||||
//! Update context, the context may be shared by different searcher/streamer
|
||||
int update_context(ContextType type, const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &metric,
|
||||
const HnswRabitqEntity::Pointer &entity,
|
||||
uint32_t magic_num);
|
||||
|
||||
inline const HnswRabitqEntity &get_entity() const {
|
||||
return *entity_;
|
||||
}
|
||||
|
||||
inline void resize_results(size_t size) {
|
||||
if (group_by_search()) {
|
||||
group_results_.resize(size);
|
||||
} else {
|
||||
results_.resize(size);
|
||||
}
|
||||
}
|
||||
|
||||
inline void topk_to_result() {
|
||||
return topk_to_result(0);
|
||||
}
|
||||
|
||||
//! Construct result from topk heap, result will be normalized
|
||||
inline void topk_to_result(uint32_t idx) {
|
||||
if (group_by_search()) {
|
||||
topk_to_group_result(idx);
|
||||
} else {
|
||||
topk_to_single_result(idx);
|
||||
}
|
||||
}
|
||||
|
||||
inline void topk_to_single_result(uint32_t idx) {
|
||||
if (force_padding_topk_ && !topk_heap_.full() &&
|
||||
topk_heap_.size() < entity_->doc_cnt()) {
|
||||
this->fill_random_to_topk_full();
|
||||
}
|
||||
if (ailego_unlikely(topk_heap_.size() == 0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
int size = std::min(topk_, static_cast<uint32_t>(topk_heap_.size()));
|
||||
topk_heap_.sort();
|
||||
results_[idx].clear();
|
||||
|
||||
for (int i = 0; i < size; ++i) {
|
||||
auto score = topk_heap_[i].second;
|
||||
if (score.est_dist > this->threshold()) {
|
||||
break;
|
||||
}
|
||||
|
||||
node_id_t id = topk_heap_[i].first;
|
||||
if (fetch_vector_) {
|
||||
results_[idx].emplace_back(entity_->get_key(id), score.est_dist, id,
|
||||
entity_->get_vector(id));
|
||||
} else {
|
||||
results_[idx].emplace_back(entity_->get_key(id), score.est_dist, id);
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
//! Construct result from topk heap, result will be normalized
|
||||
inline void topk_to_group_result(uint32_t idx) {
|
||||
ailego_assert_with(idx < group_results_.size(), "invalid idx");
|
||||
|
||||
group_results_[idx].clear();
|
||||
|
||||
std::vector<std::pair<std::string, TopkHeap>> group_topk_list;
|
||||
std::vector<std::pair<std::string, ResultRecord>> best_score_in_groups;
|
||||
for (auto itr = group_topk_heaps_.begin(); itr != group_topk_heaps_.end();
|
||||
itr++) {
|
||||
const std::string &group_id = (*itr).first;
|
||||
auto &heap = (*itr).second;
|
||||
heap.sort();
|
||||
|
||||
if (heap.size() > 0) {
|
||||
ResultRecord best_score = heap[0].second;
|
||||
best_score_in_groups.push_back(std::make_pair(group_id, best_score));
|
||||
}
|
||||
}
|
||||
|
||||
std::sort(best_score_in_groups.begin(), best_score_in_groups.end(),
|
||||
[](const std::pair<std::string, ResultRecord> &a,
|
||||
const std::pair<std::string, ResultRecord> &b) -> int {
|
||||
return a.second < b.second;
|
||||
});
|
||||
|
||||
// truncate to group num
|
||||
for (uint32_t i = 0; i < group_num() && i < best_score_in_groups.size();
|
||||
++i) {
|
||||
const std::string &group_id = best_score_in_groups[i].first;
|
||||
|
||||
group_topk_list.emplace_back(
|
||||
std::make_pair(group_id, group_topk_heaps_[group_id]));
|
||||
}
|
||||
|
||||
group_results_[idx].resize(group_topk_list.size());
|
||||
|
||||
for (uint32_t i = 0; i < group_topk_list.size(); ++i) {
|
||||
const std::string &group_id = group_topk_list[i].first;
|
||||
group_results_[idx][i].set_group_id(group_id);
|
||||
|
||||
uint32_t size = std::min(
|
||||
group_topk_, static_cast<uint32_t>(group_topk_list[i].second.size()));
|
||||
|
||||
for (uint32_t j = 0; j < size; ++j) {
|
||||
auto score = group_topk_list[i].second[j].second;
|
||||
if (score > this->threshold()) {
|
||||
break;
|
||||
}
|
||||
|
||||
node_id_t id = group_topk_list[i].second[j].first;
|
||||
|
||||
if (fetch_vector_) {
|
||||
group_results_[idx][i].mutable_docs()->emplace_back(
|
||||
entity_->get_key(id), score.est_dist, id,
|
||||
entity_->get_vector(id));
|
||||
} else {
|
||||
group_results_[idx][i].mutable_docs()->emplace_back(
|
||||
entity_->get_key(id), score.est_dist, id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline void reset_query(const void *query) {
|
||||
if (auto query_preprocess_func = index_metric_->get_query_preprocess_func();
|
||||
query_preprocess_func != nullptr) {
|
||||
size_t dim = dc().dimension();
|
||||
preprocess_buffer_.resize(dim);
|
||||
memcpy(preprocess_buffer_.data(), query, dim);
|
||||
query_preprocess_func(preprocess_buffer_.data(), dim);
|
||||
query = preprocess_buffer_.data();
|
||||
}
|
||||
|
||||
dc().reset_query(query);
|
||||
dc().clear_compare_cnt();
|
||||
query_ = query;
|
||||
}
|
||||
|
||||
inline HnswRabitqAddDistCalculator &dist_calculator() {
|
||||
return dc();
|
||||
}
|
||||
|
||||
inline TopkHeap &topk_heap() {
|
||||
return topk_heap_;
|
||||
}
|
||||
|
||||
inline TopkHeap &update_heap() {
|
||||
return update_heap_;
|
||||
}
|
||||
|
||||
inline VisitFilter &visit_filter() {
|
||||
return visit_filter_;
|
||||
}
|
||||
|
||||
inline CandidateHeap &candidates() {
|
||||
return candidates_;
|
||||
}
|
||||
|
||||
inline void set_max_scan_num(uint32_t max_scan_num) {
|
||||
max_scan_num_ = max_scan_num;
|
||||
}
|
||||
|
||||
inline void set_max_scan_limit(uint32_t max_scan_limit) {
|
||||
max_scan_limit_ = max_scan_limit;
|
||||
}
|
||||
|
||||
inline void set_min_scan_limit(uint32_t min_scan_limit) {
|
||||
min_scan_limit_ = min_scan_limit;
|
||||
}
|
||||
|
||||
inline void set_ef(uint32_t v) {
|
||||
ef_ = v;
|
||||
}
|
||||
|
||||
inline void set_filter_mode(uint32_t v) {
|
||||
filter_mode_ = v;
|
||||
}
|
||||
|
||||
inline void set_filter_negative_probability(float v) {
|
||||
negative_probability_ = v;
|
||||
}
|
||||
|
||||
inline void set_max_scan_ratio(float v) {
|
||||
max_scan_ratio_ = v;
|
||||
}
|
||||
|
||||
virtual void set_magic(uint32_t v) {
|
||||
magic_ = v;
|
||||
}
|
||||
|
||||
virtual void set_force_padding_topk(bool v) {
|
||||
force_padding_topk_ = v;
|
||||
}
|
||||
|
||||
void set_bruteforce_threshold(uint32_t v) override {
|
||||
bruteforce_threshold_ = v;
|
||||
}
|
||||
|
||||
inline uint32_t get_bruteforce_threshold() const {
|
||||
return bruteforce_threshold_;
|
||||
}
|
||||
|
||||
void set_fetch_vector(bool v) override {
|
||||
fetch_vector_ = v;
|
||||
}
|
||||
|
||||
bool fetch_vector() const override {
|
||||
return fetch_vector_;
|
||||
}
|
||||
|
||||
//! Reset context
|
||||
void reset(void) override {
|
||||
set_filter(nullptr);
|
||||
reset_threshold();
|
||||
set_fetch_vector(false);
|
||||
set_group_params(0, 0);
|
||||
reset_group_by();
|
||||
}
|
||||
|
||||
inline std::map<std::string, TopkHeap> &group_topk_heaps() {
|
||||
return group_topk_heaps_;
|
||||
}
|
||||
|
||||
inline TopkHeap &level_topk(int level) {
|
||||
if (ailego_unlikely(level_topks_.size() <= static_cast<size_t>(level))) {
|
||||
int cur_level = level_topks_.size();
|
||||
level_topks_.resize(level + 1);
|
||||
for (; cur_level <= level; ++cur_level) {
|
||||
size_t heap_size = std::max(entity_->neighbor_cnt(cur_level),
|
||||
entity_->ef_construction());
|
||||
level_topks_[cur_level].clear();
|
||||
level_topks_[cur_level].limit(heap_size);
|
||||
}
|
||||
}
|
||||
|
||||
return level_topks_[level];
|
||||
}
|
||||
|
||||
inline void check_need_adjuct_ctx(void) {
|
||||
check_need_adjuct_ctx(entity_->doc_cnt());
|
||||
}
|
||||
|
||||
inline size_t compute_reserve_cnt(uint32_t cur_doc) const {
|
||||
if (cur_doc > kMaxReserveDocCnt) {
|
||||
return kMaxReserveDocCnt;
|
||||
} else if (cur_doc < kMinReserveDocCnt) {
|
||||
return kMinReserveDocCnt;
|
||||
}
|
||||
return cur_doc;
|
||||
}
|
||||
|
||||
//! candidates heap and visitfilter need to resize as doc cnt growing up
|
||||
inline void check_need_adjuct_ctx(uint32_t doc_cnt) {
|
||||
if (ailego_unlikely(doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_)) {
|
||||
while (doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_) {
|
||||
reserve_max_doc_cnt_ =
|
||||
reserve_max_doc_cnt_ + compute_reserve_cnt(reserve_max_doc_cnt_);
|
||||
}
|
||||
uint32_t max_scan_cnt = compute_max_scan_num(reserve_max_doc_cnt_);
|
||||
max_scan_num_ = max_scan_cnt;
|
||||
visit_filter_.reset(reserve_max_doc_cnt_, max_scan_cnt);
|
||||
candidates_.clear();
|
||||
candidates_.limit(max_scan_num_);
|
||||
}
|
||||
}
|
||||
|
||||
inline uint32_t compute_max_scan_num(uint32_t max_doc_cnt) const {
|
||||
uint32_t max_scan = max_doc_cnt * max_scan_ratio_;
|
||||
if (max_scan < min_scan_limit_) {
|
||||
max_scan = min_scan_limit_;
|
||||
} else if (max_scan > max_scan_limit_) {
|
||||
max_scan = max_scan_limit_;
|
||||
}
|
||||
return max_scan;
|
||||
}
|
||||
|
||||
inline size_t get_scan_num() const {
|
||||
return dc().compare_cnt();
|
||||
}
|
||||
|
||||
inline uint64_t reach_scan_limit() const {
|
||||
return dc().compare_cnt() >= max_scan_num_;
|
||||
}
|
||||
|
||||
inline bool error() const {
|
||||
return dc().error();
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
add_dc_.clear();
|
||||
if (ailego_unlikely(this->debugging())) {
|
||||
stats_get_neighbors_cnt_ = 0u;
|
||||
stats_get_vector_cnt_ = 0u;
|
||||
stats_visit_dup_cnt_ = 0u;
|
||||
}
|
||||
// do not clear results_ for the next query will need it
|
||||
for (auto &it : results_) {
|
||||
it.clear();
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t *mutable_stats_get_neighbors() {
|
||||
return &stats_get_neighbors_cnt_;
|
||||
}
|
||||
|
||||
uint32_t *mutable_stats_get_vector() {
|
||||
return &stats_get_vector_cnt_;
|
||||
}
|
||||
|
||||
uint32_t *mutable_stats_visit_dup_cnt() {
|
||||
return &stats_visit_dup_cnt_;
|
||||
}
|
||||
|
||||
inline bool debugging(void) const {
|
||||
return debug_mode_;
|
||||
}
|
||||
|
||||
inline void update_dist_caculator_distance(
|
||||
const IndexMetric::MatrixDistance &distance,
|
||||
const IndexMetric::MatrixBatchDistance &batch_distance) {
|
||||
dc().update_distance(distance, batch_distance);
|
||||
}
|
||||
|
||||
//! Get topk
|
||||
inline uint32_t topk() const override {
|
||||
return topk_;
|
||||
}
|
||||
|
||||
//! Get group topk
|
||||
inline uint32_t group_topk() const {
|
||||
return group_topk_;
|
||||
}
|
||||
|
||||
//! Get group num
|
||||
inline uint32_t group_num() const {
|
||||
return group_num_;
|
||||
}
|
||||
|
||||
//! Get if group by search
|
||||
inline bool group_by_search() {
|
||||
return group_num_ > 0;
|
||||
}
|
||||
|
||||
//! Set group params
|
||||
void set_group_params(uint32_t group_num, uint32_t group_topk) override {
|
||||
group_num_ = group_num;
|
||||
group_topk_ = group_topk;
|
||||
|
||||
topk_ = group_topk_ * group_num_;
|
||||
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
|
||||
group_topk_heaps_.clear();
|
||||
}
|
||||
|
||||
void set_provider(IndexProvider::Pointer provider) {
|
||||
add_dc_.set_provider(std::move(provider));
|
||||
}
|
||||
|
||||
const void *query() const {
|
||||
return query_;
|
||||
}
|
||||
|
||||
private:
|
||||
inline HnswRabitqAddDistCalculator &dc() {
|
||||
return add_dc_;
|
||||
}
|
||||
|
||||
inline const HnswRabitqAddDistCalculator &dc() const {
|
||||
return add_dc_;
|
||||
}
|
||||
|
||||
private:
|
||||
// Filling random nodes if topk not full
|
||||
void fill_random_to_topk_full(void);
|
||||
|
||||
constexpr static uint32_t kTriggerReserveCnt = 4096UL;
|
||||
constexpr static uint32_t kMinReserveDocCnt = 4096UL;
|
||||
constexpr static uint32_t kMaxReserveDocCnt = 128 * 1024UL;
|
||||
constexpr static uint32_t kInvalidMgic = -1U;
|
||||
|
||||
private:
|
||||
HnswRabitqEntity::Pointer entity_;
|
||||
HnswRabitqAddDistCalculator add_dc_;
|
||||
IndexMetric::Pointer metric_;
|
||||
|
||||
bool debug_mode_{false};
|
||||
bool force_padding_topk_{false};
|
||||
uint32_t max_scan_num_{0};
|
||||
uint32_t max_scan_limit_{0};
|
||||
uint32_t min_scan_limit_{0};
|
||||
uint32_t reserve_max_doc_cnt_{kMinReserveDocCnt};
|
||||
uint32_t topk_{0};
|
||||
uint32_t group_topk_{0};
|
||||
uint32_t filter_mode_{VisitFilter::ByteMap};
|
||||
float negative_probability_{HnswRabitqEntity::kDefaultBFNegativeProbability};
|
||||
uint32_t ef_{HnswRabitqEntity::kDefaultEf};
|
||||
float max_scan_ratio_{HnswRabitqEntity::kDefaultScanRatio};
|
||||
uint32_t magic_{0U};
|
||||
std::vector<IndexDocumentList> results_{};
|
||||
std::vector<IndexGroupDocumentList> group_results_{};
|
||||
TopkHeap topk_heap_{};
|
||||
TopkHeap update_heap_{};
|
||||
std::vector<TopkHeap> level_topks_{};
|
||||
CandidateHeap candidates_{};
|
||||
VisitFilter visit_filter_{};
|
||||
uint32_t bruteforce_threshold_{};
|
||||
bool fetch_vector_{false};
|
||||
|
||||
uint32_t group_num_{0};
|
||||
std::map<std::string, TopkHeap> group_topk_heaps_{};
|
||||
|
||||
uint32_t type_{kUnknownContext};
|
||||
//! debug stats info
|
||||
uint32_t stats_get_neighbors_cnt_{0u};
|
||||
uint32_t stats_get_vector_cnt_{0u};
|
||||
uint32_t stats_visit_dup_cnt_{0u};
|
||||
std::string preprocess_buffer_;
|
||||
const void *query_{nullptr};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2025-present the centaurdb project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
|
||||
#include "core/algorithm/hnsw_rabitq/hnsw_rabitq_dist_calculator.h"
|
||||
#include "zvec/core/framework/index_error.h"
|
||||
|
||||
namespace zvec::core {
|
||||
|
||||
int HnswRabitqAddDistCalculator::get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
const node_id_t id = ids[i];
|
||||
key_t key = entity_->get_key(id);
|
||||
if (key == kInvalidKey) {
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
IndexStorage::MemoryBlock block;
|
||||
int ret = provider_->get_vector(key, block);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
vec_blocks.push_back(std::move(block));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace zvec::core
|
||||
@@ -0,0 +1,240 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "zvec/core/framework/index_meta.h"
|
||||
#include "zvec/core/framework/index_metric.h"
|
||||
#include "zvec/core/framework/index_provider.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! HnswRabitqAddDistCalculator is only used for index construction
|
||||
class HnswRabitqAddDistCalculator {
|
||||
public:
|
||||
typedef std::shared_ptr<HnswRabitqAddDistCalculator> Pointer;
|
||||
|
||||
public:
|
||||
enum DistType {
|
||||
DIST_NONE = 0,
|
||||
DIST_DENSE = 1,
|
||||
DIST_HYBRID = 2,
|
||||
DIST_SPARSE = 3
|
||||
};
|
||||
|
||||
public:
|
||||
//! Constructor
|
||||
HnswRabitqAddDistCalculator(const HnswRabitqEntity *entity,
|
||||
const IndexMetric::Pointer &metric, uint32_t dim)
|
||||
: entity_(entity),
|
||||
distance_(metric->distance()),
|
||||
batch_distance_(metric->batch_distance()),
|
||||
query_(nullptr),
|
||||
dim_(dim),
|
||||
compare_cnt_(0) {}
|
||||
|
||||
//! Constructor
|
||||
HnswRabitqAddDistCalculator(const HnswRabitqEntity *entity,
|
||||
const IndexMetric::Pointer &metric, uint32_t dim,
|
||||
const void *query)
|
||||
: entity_(entity),
|
||||
distance_(metric->distance()),
|
||||
batch_distance_(metric->batch_distance()),
|
||||
query_(query),
|
||||
dim_(dim),
|
||||
compare_cnt_(0) {}
|
||||
|
||||
//! Constructor
|
||||
HnswRabitqAddDistCalculator(const HnswRabitqEntity *entity,
|
||||
const IndexMetric::Pointer &metric)
|
||||
: entity_(entity),
|
||||
distance_(metric->distance()),
|
||||
batch_distance_(metric->batch_distance()),
|
||||
query_(nullptr),
|
||||
dim_(0),
|
||||
compare_cnt_(0) {}
|
||||
|
||||
void update(const HnswRabitqEntity *entity,
|
||||
const IndexMetric::Pointer &metric) {
|
||||
entity_ = entity;
|
||||
distance_ = metric->distance();
|
||||
batch_distance_ = metric->batch_distance();
|
||||
}
|
||||
|
||||
void update(const HnswRabitqEntity *entity,
|
||||
const IndexMetric::Pointer &metric, uint32_t dim) {
|
||||
entity_ = entity;
|
||||
distance_ = metric->distance();
|
||||
batch_distance_ = metric->batch_distance();
|
||||
dim_ = dim;
|
||||
}
|
||||
|
||||
inline void update_distance(
|
||||
const IndexMetric::MatrixDistance &distance,
|
||||
const IndexMetric::MatrixBatchDistance &batch_distance) {
|
||||
distance_ = distance;
|
||||
batch_distance_ = batch_distance;
|
||||
}
|
||||
|
||||
//! Reset query vector data
|
||||
inline void reset_query(const void *query) {
|
||||
error_ = false;
|
||||
query_ = query;
|
||||
}
|
||||
|
||||
//! Returns distance
|
||||
inline dist_t dist(const void *vec_lhs, const void *vec_rhs) {
|
||||
if (ailego_unlikely(vec_lhs == nullptr || vec_rhs == nullptr)) {
|
||||
LOG_ERROR("Nullptr of dense vector");
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
float score{0.0f};
|
||||
|
||||
distance_(vec_lhs, vec_rhs, dim_, &score);
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
//! Returns distance between query and vec.
|
||||
inline dist_t dist(const void *vec) {
|
||||
compare_cnt_++;
|
||||
|
||||
return dist(vec, query_);
|
||||
}
|
||||
|
||||
//! Return distance between query and node id.
|
||||
inline dist_t dist(node_id_t id) {
|
||||
compare_cnt_++;
|
||||
|
||||
const void *feat = get_vector(id);
|
||||
if (ailego_unlikely(feat == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, id=%u", id);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
return dist(feat, query_);
|
||||
}
|
||||
|
||||
//! Return dist node lhs between node rhs
|
||||
inline dist_t dist(node_id_t lhs, node_id_t rhs) {
|
||||
compare_cnt_++;
|
||||
|
||||
const void *feat = get_vector(lhs);
|
||||
const void *query = get_vector(rhs);
|
||||
if (ailego_unlikely(feat == nullptr || query == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector");
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
|
||||
return dist(feat, query);
|
||||
}
|
||||
|
||||
dist_t operator()(const void *vec) {
|
||||
return dist(vec);
|
||||
}
|
||||
|
||||
dist_t operator()(id_t i) {
|
||||
return dist(i);
|
||||
}
|
||||
|
||||
dist_t operator()(id_t lhs, id_t rhs) {
|
||||
return dist(lhs, rhs);
|
||||
}
|
||||
|
||||
void batch_dist(const void **vecs, size_t num, dist_t *distances) {
|
||||
compare_cnt_++;
|
||||
|
||||
batch_distance_(vecs, query_, num, dim_, distances);
|
||||
}
|
||||
|
||||
inline dist_t batch_dist(node_id_t id) {
|
||||
compare_cnt_++;
|
||||
|
||||
const void *feat = get_vector(id);
|
||||
if (ailego_unlikely(feat == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, id=%u", id);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
dist_t score = 0;
|
||||
batch_distance_(&feat, query_, 1, dim_, &score);
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
compare_cnt_ = 0;
|
||||
error_ = false;
|
||||
}
|
||||
|
||||
inline void clear_compare_cnt() {
|
||||
compare_cnt_ = 0;
|
||||
}
|
||||
|
||||
inline bool error() const {
|
||||
return error_;
|
||||
}
|
||||
|
||||
//! Get distances compute times
|
||||
inline uint32_t compare_cnt() const {
|
||||
return compare_cnt_;
|
||||
}
|
||||
|
||||
inline uint32_t dimension() const {
|
||||
return dim_;
|
||||
}
|
||||
|
||||
void set_provider(IndexProvider::Pointer provider) {
|
||||
provider_ = std::move(provider);
|
||||
}
|
||||
|
||||
int get_vector(const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const;
|
||||
|
||||
const void *get_vector(node_id_t id) const {
|
||||
key_t key = entity_->get_key(id);
|
||||
if (key == kInvalidKey) {
|
||||
return nullptr;
|
||||
}
|
||||
return provider_->get_vector(key);
|
||||
}
|
||||
|
||||
private:
|
||||
HnswRabitqAddDistCalculator(const HnswRabitqAddDistCalculator &) = delete;
|
||||
HnswRabitqAddDistCalculator &operator=(const HnswRabitqAddDistCalculator &) =
|
||||
delete;
|
||||
|
||||
private:
|
||||
const HnswRabitqEntity *entity_;
|
||||
IndexMetric::MatrixDistance distance_;
|
||||
IndexMetric::MatrixBatchDistance batch_distance_;
|
||||
|
||||
const void *query_;
|
||||
uint32_t dim_;
|
||||
|
||||
uint32_t compare_cnt_; // record distance compute times
|
||||
uint32_t compare_cnt_batch_; // record batch distance compute time
|
||||
bool error_{false};
|
||||
|
||||
// get raw vector
|
||||
IndexProvider::Pointer provider_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,382 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
#include <rabitqlib/index/query.hpp>
|
||||
#include "utility/sparse_utility.h"
|
||||
#include "zvec/core/framework/index_stats.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
const std::string HnswRabitqEntity::kGraphHeaderSegmentId = "graph.header";
|
||||
const std::string HnswRabitqEntity::kGraphFeaturesSegmentId = "graph.features";
|
||||
const std::string HnswRabitqEntity::kGraphKeysSegmentId = "graph.keys";
|
||||
const std::string HnswRabitqEntity::kGraphNeighborsSegmentId =
|
||||
"graph.neighbors";
|
||||
const std::string HnswRabitqEntity::kGraphOffsetsSegmentId = "graph.offsets";
|
||||
const std::string HnswRabitqEntity::kGraphMappingSegmentId = "graph.mapping";
|
||||
const std::string HnswRabitqEntity::kHnswHeaderSegmentId = "hnsw.header";
|
||||
const std::string HnswRabitqEntity::kHnswNeighborsSegmentId = "hnsw.neighbors";
|
||||
const std::string HnswRabitqEntity::kHnswOffsetsSegmentId = "hnsw.offsets";
|
||||
|
||||
void HnswRabitqEntity::update_rabitq_params_and_vector_size(
|
||||
uint32_t dimension) {
|
||||
uint32_t padded_dim = ((dimension + 63) / 64) * 64;
|
||||
header_.graph.padded_dim = padded_dim;
|
||||
// BinDataMap layout: bin_code (padded_dim/8) + f_add + f_rescale + f_error
|
||||
header_.graph.size_bin_data =
|
||||
rabitqlib::BinDataMap<float>::data_bytes(padded_dim);
|
||||
// ExDataMap layout: ex_code (padded_dim*ex_bits/8) + f_add_ex + f_rescale_ex
|
||||
header_.graph.size_ex_data = rabitqlib::ExDataMap<float>::data_bytes(
|
||||
padded_dim, header_.graph.ex_bits);
|
||||
// quantized vector format: cluster_id + bin_data + ex_data
|
||||
header_.graph.vector_size =
|
||||
sizeof(uint32_t) + size_bin_data() + size_ex_data();
|
||||
}
|
||||
|
||||
int HnswRabitqEntity::CalcAndAddPadding(const IndexDumper::Pointer &dumper,
|
||||
size_t data_size,
|
||||
size_t *padding_size) {
|
||||
*padding_size = AlignSize(data_size) - data_size;
|
||||
if (*padding_size == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::string padding(*padding_size, '\0');
|
||||
if (dumper->write(padding.data(), *padding_size) != *padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %zu", *padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int64_t HnswRabitqEntity::dump_segment(const IndexDumper::Pointer &dumper,
|
||||
const std::string &segment_id,
|
||||
const void *data, size_t size) const {
|
||||
size_t len = dumper->write(data, size);
|
||||
if (len != size) {
|
||||
LOG_ERROR("Dump segment %s data failed, expect: %zu, actual: %zu",
|
||||
segment_id.c_str(), size, len);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
size_t padding_size = AlignSize(size) - size;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %zu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t crc = ailego::Crc32c::Hash(data, size);
|
||||
int ret = dumper->append(segment_id, size, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return len + padding_size;
|
||||
}
|
||||
|
||||
int64_t HnswRabitqEntity::dump_header(const IndexDumper::Pointer &dumper,
|
||||
const HNSWHeader &hd) const {
|
||||
//! dump basic graph header. header is aligned and does not need padding
|
||||
int64_t graph_hd_size =
|
||||
dump_segment(dumper, kGraphHeaderSegmentId, &hd.graph, hd.graph.size);
|
||||
if (graph_hd_size < 0) {
|
||||
return graph_hd_size;
|
||||
}
|
||||
|
||||
//! dump basic graph header. header is aligned and does not need padding
|
||||
int64_t hnsw_hd_size =
|
||||
dump_segment(dumper, kHnswHeaderSegmentId, &hd.hnsw, hd.hnsw.size);
|
||||
if (hnsw_hd_size < 0) {
|
||||
return hnsw_hd_size;
|
||||
}
|
||||
|
||||
return graph_hd_size + hnsw_hd_size;
|
||||
}
|
||||
|
||||
void HnswRabitqEntity::reshuffle_vectors(
|
||||
const std::function<level_t(node_id_t)> & /*get_level*/,
|
||||
std::vector<node_id_t> * /*n2o_mapping*/,
|
||||
std::vector<node_id_t> * /*o2n_mapping*/, key_t * /*keys*/) const {
|
||||
// TODO
|
||||
return;
|
||||
}
|
||||
|
||||
int64_t HnswRabitqEntity::dump_mapping_segment(
|
||||
const IndexDumper::Pointer &dumper, const key_t *keys) const {
|
||||
std::vector<node_id_t> mapping(doc_cnt());
|
||||
|
||||
std::iota(mapping.begin(), mapping.end(), 0U);
|
||||
std::sort(mapping.begin(), mapping.end(),
|
||||
[&](node_id_t i, node_id_t j) { return keys[i] < keys[j]; });
|
||||
|
||||
size_t size = mapping.size() * sizeof(node_id_t);
|
||||
|
||||
return dump_segment(dumper, kGraphMappingSegmentId, mapping.data(), size);
|
||||
}
|
||||
|
||||
int64_t HnswRabitqEntity::dump_segments(
|
||||
const IndexDumper::Pointer &dumper, key_t *keys,
|
||||
const std::function<level_t(node_id_t)> &get_level) const {
|
||||
HNSWHeader dump_hd(header());
|
||||
|
||||
dump_hd.graph.node_size = AlignSize(vector_size());
|
||||
|
||||
std::vector<node_id_t> n2o_mapping; // map new id to origin id
|
||||
std::vector<node_id_t> o2n_mapping; // map origin id to new id
|
||||
reshuffle_vectors(get_level, &n2o_mapping, &o2n_mapping, keys);
|
||||
if (!o2n_mapping.empty()) {
|
||||
dump_hd.hnsw.entry_point = o2n_mapping[entry_point()];
|
||||
}
|
||||
|
||||
//! Dump header
|
||||
int64_t hd_size = dump_header(dumper, dump_hd);
|
||||
if (hd_size < 0) {
|
||||
return hd_size;
|
||||
}
|
||||
|
||||
//! Dump vectors
|
||||
int64_t vecs_size = dump_vectors(dumper, n2o_mapping);
|
||||
if (vecs_size < 0) {
|
||||
return vecs_size;
|
||||
}
|
||||
|
||||
//! Dump neighbors
|
||||
auto neighbors_size =
|
||||
dump_neighbors(dumper, get_level, n2o_mapping, o2n_mapping);
|
||||
if (neighbors_size < 0) {
|
||||
return neighbors_size;
|
||||
}
|
||||
//! free memory
|
||||
n2o_mapping = std::vector<node_id_t>();
|
||||
o2n_mapping = std::vector<node_id_t>();
|
||||
|
||||
//! Dump keys
|
||||
size_t key_segment_size = doc_cnt() * sizeof(key_t);
|
||||
int64_t keys_size =
|
||||
dump_segment(dumper, kGraphKeysSegmentId, keys, key_segment_size);
|
||||
if (keys_size < 0) {
|
||||
return keys_size;
|
||||
}
|
||||
|
||||
//! Dump mapping
|
||||
int64_t mapping_size = dump_mapping_segment(dumper, keys);
|
||||
if (mapping_size < 0) {
|
||||
return mapping_size;
|
||||
}
|
||||
|
||||
return hd_size + keys_size + vecs_size + neighbors_size + mapping_size;
|
||||
}
|
||||
|
||||
int64_t HnswRabitqEntity::dump_vectors(
|
||||
const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping) const {
|
||||
size_t vector_dump_size = vector_size();
|
||||
|
||||
size_t padding_size = AlignSize(vector_dump_size) - vector_dump_size;
|
||||
|
||||
char padding[padding_size];
|
||||
memset(padding, 0, sizeof(padding));
|
||||
const void *data = nullptr;
|
||||
uint32_t crc = 0U;
|
||||
size_t vecs_size = 0UL;
|
||||
|
||||
//! dump vectors
|
||||
for (node_id_t id = 0; id < doc_cnt(); ++id) {
|
||||
data = get_vector(reorder_mapping.empty() ? id : reorder_mapping[id]);
|
||||
if (ailego_unlikely(!data)) {
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
size_t len = dumper->write(data, vector_size());
|
||||
if (len != vector_size()) {
|
||||
LOG_ERROR("Dump vectors failed, write=%zu expect=%zu", len,
|
||||
vector_size());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
crc = ailego::Crc32c::Hash(data, vector_size(), crc);
|
||||
vecs_size += vector_size();
|
||||
|
||||
if (padding_size == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
len = dumper->write(padding, padding_size);
|
||||
if (len != padding_size) {
|
||||
LOG_ERROR("Dump vectors failed, write=%zu expect=%zu", len, padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(padding, padding_size, crc);
|
||||
vecs_size += padding_size;
|
||||
}
|
||||
|
||||
int ret = dumper->append(kGraphFeaturesSegmentId, vecs_size, 0UL, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump vectors segment meta failed, ret %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return vecs_size;
|
||||
}
|
||||
|
||||
int64_t HnswRabitqEntity::dump_graph_neighbors(
|
||||
const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping,
|
||||
const std::vector<node_id_t> &neighbor_mapping) const {
|
||||
std::vector<GraphNeighborMeta> graph_meta;
|
||||
graph_meta.reserve(doc_cnt());
|
||||
size_t offset = 0;
|
||||
uint32_t crc = 0;
|
||||
node_id_t mapping[l0_neighbor_cnt()];
|
||||
|
||||
uint32_t min_neighbor_count = 10000;
|
||||
uint32_t max_neighbor_count = 0;
|
||||
size_t sum_neighbor_count = 0;
|
||||
|
||||
for (node_id_t id = 0; id < doc_cnt(); ++id) {
|
||||
const Neighbors neighbors =
|
||||
get_neighbors(0, reorder_mapping.empty() ? id : reorder_mapping[id]);
|
||||
ailego_assert_with(!!neighbors.data, "invalid neighbors");
|
||||
ailego_assert_with(neighbors.size() <= l0_neighbor_cnt(),
|
||||
"invalid neighbors");
|
||||
|
||||
uint32_t neighbor_count = neighbors.size();
|
||||
if (neighbor_count < min_neighbor_count) {
|
||||
min_neighbor_count = neighbor_count;
|
||||
}
|
||||
if (neighbor_count > max_neighbor_count) {
|
||||
max_neighbor_count = neighbor_count;
|
||||
}
|
||||
sum_neighbor_count += neighbor_count;
|
||||
|
||||
graph_meta.emplace_back(offset, neighbor_count);
|
||||
size_t size = neighbors.size() * sizeof(node_id_t);
|
||||
const node_id_t *data = &neighbors[0];
|
||||
if (!neighbor_mapping.empty()) {
|
||||
for (node_id_t i = 0; i < neighbors.size(); ++i) {
|
||||
mapping[i] = neighbor_mapping[neighbors[i]];
|
||||
}
|
||||
data = mapping;
|
||||
}
|
||||
if (dumper->write(data, size) != size) {
|
||||
LOG_ERROR("Dump graph neighbor id=%zu failed, size %zu",
|
||||
static_cast<size_t>(id), size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(data, size, crc);
|
||||
offset += size;
|
||||
}
|
||||
|
||||
uint32_t average_neighbor_count = 0;
|
||||
if (doc_cnt() > 0) {
|
||||
average_neighbor_count = sum_neighbor_count / doc_cnt();
|
||||
}
|
||||
LOG_INFO(
|
||||
"Dump hnsw graph: min_neighbor_count[%u] max_neighbor_count[%u] "
|
||||
"average_neighbor_count[%u]",
|
||||
min_neighbor_count, max_neighbor_count, average_neighbor_count);
|
||||
|
||||
size_t padding_size = 0;
|
||||
int ret = CalcAndAddPadding(dumper, offset, &padding_size);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
ret = dumper->append(kGraphNeighborsSegmentId, offset, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump segment %s failed, ret %d",
|
||||
kGraphNeighborsSegmentId.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! dump level 0 neighbors meta
|
||||
auto len = dump_segment(dumper, kGraphOffsetsSegmentId, graph_meta.data(),
|
||||
graph_meta.size() * sizeof(GraphNeighborMeta));
|
||||
if (len < 0) {
|
||||
return len;
|
||||
}
|
||||
|
||||
return len + offset + padding_size;
|
||||
}
|
||||
|
||||
int64_t HnswRabitqEntity::dump_upper_neighbors(
|
||||
const IndexDumper::Pointer &dumper,
|
||||
const std::function<level_t(node_id_t)> &get_level,
|
||||
const std::vector<node_id_t> &reorder_mapping,
|
||||
const std::vector<node_id_t> &neighbor_mapping) const {
|
||||
std::vector<HnswNeighborMeta> hnsw_meta;
|
||||
hnsw_meta.reserve(doc_cnt());
|
||||
size_t offset = 0;
|
||||
uint32_t crc = 0;
|
||||
node_id_t buffer[upper_neighbor_cnt() + 1];
|
||||
for (node_id_t id = 0; id < doc_cnt(); ++id) {
|
||||
node_id_t new_id = reorder_mapping.empty() ? id : reorder_mapping[id];
|
||||
auto level = get_level(new_id);
|
||||
if (level == 0) {
|
||||
hnsw_meta.emplace_back(0U, 0U);
|
||||
continue;
|
||||
}
|
||||
hnsw_meta.emplace_back(offset, level);
|
||||
ailego_assert_with((size_t)level < kMaxGraphLayers, "invalid level");
|
||||
for (level_t cur_level = 1; cur_level <= level; ++cur_level) {
|
||||
const Neighbors neighbors = get_neighbors(cur_level, new_id);
|
||||
ailego_assert_with(!!neighbors.data, "invalid neighbors");
|
||||
ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level),
|
||||
"invalid neighbors");
|
||||
memset(buffer, 0, sizeof(buffer));
|
||||
buffer[0] = neighbors.size();
|
||||
if (neighbor_mapping.empty()) {
|
||||
memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t));
|
||||
} else {
|
||||
for (node_id_t i = 0; i < neighbors.size(); ++i) {
|
||||
buffer[i + 1] = neighbor_mapping[neighbors[i]];
|
||||
}
|
||||
}
|
||||
if (dumper->write(buffer, sizeof(buffer)) != sizeof(buffer)) {
|
||||
LOG_ERROR("Dump graph neighbor id=%zu failed, size %zu",
|
||||
static_cast<size_t>(id), sizeof(buffer));
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(buffer, sizeof(buffer), crc);
|
||||
offset += sizeof(buffer);
|
||||
}
|
||||
}
|
||||
size_t padding_size = 0;
|
||||
int ret = CalcAndAddPadding(dumper, offset, &padding_size);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = dumper->append(kHnswNeighborsSegmentId, offset, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump segment %s failed, ret %d", kHnswNeighborsSegmentId.c_str(),
|
||||
ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! dump level 0 neighbors meta
|
||||
auto len = dump_segment(dumper, kHnswOffsetsSegmentId, hnsw_meta.data(),
|
||||
hnsw_meta.size() * sizeof(HnswNeighborMeta));
|
||||
if (len < 0) {
|
||||
return len;
|
||||
}
|
||||
|
||||
return len + offset + padding_size;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,614 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <execinfo.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <string>
|
||||
#include <ailego/utility/memory_helper.h>
|
||||
#include <zvec/ailego/container/heap.h>
|
||||
#include <zvec/ailego/logger/logger.h>
|
||||
#include "zvec/core/framework/index_dumper.h"
|
||||
#include "zvec/core/framework/index_error.h"
|
||||
#include "zvec/core/framework/index_storage.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
using node_id_t = uint32_t;
|
||||
using key_t = uint64_t;
|
||||
using level_t = int32_t;
|
||||
using dist_t = float;
|
||||
struct EstimateRecord {
|
||||
float ip_x0_qr;
|
||||
float est_dist;
|
||||
float low_dist;
|
||||
|
||||
bool operator<(const EstimateRecord &other) const {
|
||||
return this->est_dist < other.est_dist;
|
||||
}
|
||||
};
|
||||
struct ResultRecord {
|
||||
float est_dist;
|
||||
float low_dist;
|
||||
ResultRecord() : est_dist(0.0f), low_dist(0.0f) {}
|
||||
ResultRecord(float dist) : est_dist(dist), low_dist(dist) {}
|
||||
explicit ResultRecord(const EstimateRecord &other)
|
||||
: est_dist(other.est_dist), low_dist(other.low_dist) {}
|
||||
ResultRecord(float est_dist, float low_dist)
|
||||
: est_dist(est_dist), low_dist(low_dist) {}
|
||||
bool operator<(const ResultRecord &other) const {
|
||||
return this->est_dist < other.est_dist;
|
||||
}
|
||||
bool operator<=(const ResultRecord &other) const {
|
||||
return this->est_dist <= other.est_dist;
|
||||
}
|
||||
bool operator>(const ResultRecord &other) const {
|
||||
return this->est_dist > other.est_dist;
|
||||
}
|
||||
};
|
||||
using TopkHeap = ailego::KeyValueHeap<node_id_t, ResultRecord>;
|
||||
using CandidateHeap =
|
||||
ailego::KeyValueHeap<node_id_t, ResultRecord, std::greater<ResultRecord>>;
|
||||
constexpr node_id_t kInvalidNodeId = static_cast<node_id_t>(-1);
|
||||
constexpr key_t kInvalidKey = static_cast<key_t>(-1);
|
||||
class DistCalculator;
|
||||
|
||||
struct GraphHeader {
|
||||
uint32_t size;
|
||||
uint32_t version;
|
||||
uint32_t graph_type;
|
||||
uint32_t doc_count;
|
||||
uint32_t vector_size;
|
||||
uint32_t node_size;
|
||||
uint32_t l0_neighbor_count;
|
||||
uint32_t prune_type;
|
||||
uint32_t prune_neighbor_count;
|
||||
uint32_t ef_construction;
|
||||
uint32_t options;
|
||||
uint32_t min_neighbor_count;
|
||||
uint32_t padded_dim;
|
||||
uint32_t size_bin_data;
|
||||
uint32_t size_ex_data;
|
||||
uint8_t ex_bits;
|
||||
uint8_t reserved_[4067];
|
||||
};
|
||||
|
||||
static_assert(sizeof(GraphHeader) % 32 == 0,
|
||||
"GraphHeader must be aligned with 32 bytes");
|
||||
|
||||
//! Hnsw upper neighbor header
|
||||
struct HnswHeader {
|
||||
uint32_t size; // header size
|
||||
uint32_t revision; // current total docs of the graph
|
||||
uint32_t upper_neighbor_count;
|
||||
uint32_t ef_construction;
|
||||
uint32_t scaling_factor;
|
||||
uint32_t max_level;
|
||||
uint32_t entry_point;
|
||||
uint32_t options;
|
||||
uint8_t reserved_[30];
|
||||
};
|
||||
|
||||
static_assert(sizeof(HnswHeader) % 32 == 0,
|
||||
"GraphHeader must be aligned with 32 bytes");
|
||||
|
||||
//! Hnsw common header and upper neighbor header
|
||||
struct HNSWHeader {
|
||||
HNSWHeader() {
|
||||
clear();
|
||||
}
|
||||
|
||||
HNSWHeader(const HNSWHeader &header) {
|
||||
memcpy(this, &header, sizeof(header));
|
||||
}
|
||||
|
||||
HNSWHeader &operator=(const HNSWHeader &header) {
|
||||
memcpy(this, &header, sizeof(header));
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Reset state to zero, and the params is untouched
|
||||
void inline reset() {
|
||||
graph.doc_count = 0U;
|
||||
hnsw.entry_point = kInvalidNodeId;
|
||||
hnsw.max_level = 0;
|
||||
}
|
||||
|
||||
//! Clear all fields to init value
|
||||
void inline clear() {
|
||||
memset(this, 0, sizeof(HNSWHeader));
|
||||
hnsw.entry_point = kInvalidNodeId;
|
||||
graph.size = sizeof(GraphHeader);
|
||||
hnsw.size = sizeof(HnswHeader);
|
||||
}
|
||||
|
||||
size_t l0_neighbor_cnt() const {
|
||||
return graph.l0_neighbor_count;
|
||||
}
|
||||
|
||||
size_t upper_neighbor_cnt() const {
|
||||
return hnsw.upper_neighbor_count;
|
||||
}
|
||||
|
||||
size_t vector_size() const {
|
||||
return graph.vector_size;
|
||||
}
|
||||
|
||||
uint8_t ex_bits() const {
|
||||
return graph.ex_bits;
|
||||
}
|
||||
|
||||
uint32_t padded_dim() const {
|
||||
return graph.padded_dim;
|
||||
}
|
||||
|
||||
size_t ef_construction() const {
|
||||
return graph.ef_construction;
|
||||
}
|
||||
|
||||
size_t scaling_factor() const {
|
||||
return hnsw.scaling_factor;
|
||||
}
|
||||
|
||||
size_t neighbor_prune_cnt() const {
|
||||
return graph.prune_neighbor_count;
|
||||
}
|
||||
|
||||
node_id_t entry_point() const {
|
||||
return hnsw.entry_point;
|
||||
}
|
||||
|
||||
node_id_t doc_cnt() const {
|
||||
return graph.doc_count;
|
||||
}
|
||||
|
||||
GraphHeader graph;
|
||||
HnswHeader hnsw;
|
||||
};
|
||||
|
||||
struct NeighborsHeader {
|
||||
uint32_t neighbor_cnt;
|
||||
node_id_t neighbors[0];
|
||||
};
|
||||
|
||||
struct Neighbors {
|
||||
Neighbors() : cnt{0}, data{nullptr} {}
|
||||
|
||||
Neighbors(uint32_t cnt_in, const node_id_t *data_in)
|
||||
: cnt{cnt_in}, data{data_in} {}
|
||||
|
||||
Neighbors(IndexStorage::MemoryBlock &&mem_block)
|
||||
: neighbor_block{std::move(mem_block)} {
|
||||
auto hd = reinterpret_cast<const NeighborsHeader *>(neighbor_block.data());
|
||||
cnt = hd->neighbor_cnt;
|
||||
data = hd->neighbors;
|
||||
}
|
||||
|
||||
size_t size(void) const {
|
||||
return cnt;
|
||||
}
|
||||
|
||||
const node_id_t &operator[](size_t idx) const {
|
||||
return data[idx];
|
||||
}
|
||||
|
||||
uint32_t cnt;
|
||||
const node_id_t *data;
|
||||
IndexStorage::MemoryBlock neighbor_block;
|
||||
};
|
||||
|
||||
//! level 0 neighbors offset
|
||||
struct GraphNeighborMeta {
|
||||
GraphNeighborMeta(size_t o, size_t cnt) : offset(o), neighbor_cnt(cnt) {}
|
||||
|
||||
uint64_t offset : 48;
|
||||
uint64_t neighbor_cnt : 16;
|
||||
};
|
||||
|
||||
//! hnsw upper neighbors meta
|
||||
struct HnswNeighborMeta {
|
||||
HnswNeighborMeta(size_t o, size_t l) : offset(o), level(l) {}
|
||||
|
||||
uint64_t offset : 48; // offset = idx * upper neighors size
|
||||
uint64_t level : 16;
|
||||
};
|
||||
|
||||
class HnswRabitqEntity {
|
||||
public:
|
||||
//! Constructor
|
||||
HnswRabitqEntity() {}
|
||||
|
||||
//! Constructor
|
||||
HnswRabitqEntity(const HNSWHeader &hd) {
|
||||
header_ = hd;
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
virtual ~HnswRabitqEntity() {}
|
||||
|
||||
//! HnswRabitqEntity Pointerd;
|
||||
typedef std::shared_ptr<HnswRabitqEntity> Pointer;
|
||||
|
||||
//! Get max neighbor size of graph level
|
||||
inline size_t neighbor_cnt(level_t level) const {
|
||||
return level == 0 ? header_.graph.l0_neighbor_count
|
||||
: header_.hnsw.upper_neighbor_count;
|
||||
}
|
||||
|
||||
//! get max neighbor size of graph level 0
|
||||
inline size_t l0_neighbor_cnt() const {
|
||||
return header_.graph.l0_neighbor_count;
|
||||
}
|
||||
|
||||
//! get min neighbor size of graph
|
||||
inline size_t min_neighbor_cnt() const {
|
||||
return header_.graph.min_neighbor_count;
|
||||
}
|
||||
|
||||
//! get upper neighbor size of graph level other than 0
|
||||
inline size_t upper_neighbor_cnt() const {
|
||||
return header_.hnsw.upper_neighbor_count;
|
||||
}
|
||||
|
||||
//! Get current total doc of the hnsw graph
|
||||
inline node_id_t *mutable_doc_cnt() {
|
||||
return &header_.graph.doc_count;
|
||||
}
|
||||
|
||||
inline node_id_t doc_cnt() const {
|
||||
return header_.graph.doc_count;
|
||||
}
|
||||
|
||||
//! Get hnsw graph scaling params
|
||||
inline size_t scaling_factor() const {
|
||||
return header_.hnsw.scaling_factor;
|
||||
}
|
||||
|
||||
//! Get prune_size
|
||||
inline size_t prune_cnt() const {
|
||||
return header_.graph.prune_neighbor_count;
|
||||
}
|
||||
|
||||
//! Current entity of top level graph
|
||||
inline node_id_t entry_point() const {
|
||||
return header_.hnsw.entry_point;
|
||||
}
|
||||
|
||||
//! Current max graph level
|
||||
inline level_t cur_max_level() const {
|
||||
return header_.hnsw.max_level;
|
||||
}
|
||||
|
||||
//! Retrieve index vector size
|
||||
size_t vector_size() const {
|
||||
return header_.graph.vector_size;
|
||||
}
|
||||
|
||||
//! Retrieve node size
|
||||
size_t node_size() const {
|
||||
return header_.graph.node_size;
|
||||
}
|
||||
|
||||
//! Retrieve ef constuction
|
||||
size_t ef_construction() const {
|
||||
return header_.graph.ef_construction;
|
||||
}
|
||||
|
||||
uint8_t ex_bits() const {
|
||||
return header_.graph.ex_bits;
|
||||
}
|
||||
|
||||
uint32_t padded_dim() const {
|
||||
return header_.graph.padded_dim;
|
||||
}
|
||||
|
||||
uint32_t size_bin_data() const {
|
||||
return header_.graph.size_bin_data;
|
||||
}
|
||||
|
||||
uint32_t size_ex_data() const {
|
||||
return header_.graph.size_ex_data;
|
||||
}
|
||||
|
||||
void update_rabitq_params_and_vector_size(uint32_t dimension);
|
||||
|
||||
void set_ex_bits(uint8_t ex_bits) {
|
||||
header_.graph.ex_bits = ex_bits;
|
||||
}
|
||||
|
||||
void set_prune_cnt(size_t v) {
|
||||
header_.graph.prune_neighbor_count = v;
|
||||
}
|
||||
|
||||
void set_scaling_factor(size_t val) {
|
||||
header_.hnsw.scaling_factor = val;
|
||||
}
|
||||
|
||||
void set_l0_neighbor_cnt(size_t cnt) {
|
||||
header_.graph.l0_neighbor_count = cnt;
|
||||
}
|
||||
|
||||
void set_min_neighbor_cnt(size_t cnt) {
|
||||
header_.graph.min_neighbor_count = cnt;
|
||||
}
|
||||
|
||||
void set_upper_neighbor_cnt(size_t cnt) {
|
||||
header_.hnsw.upper_neighbor_count = cnt;
|
||||
}
|
||||
|
||||
void set_ef_construction(size_t ef) {
|
||||
header_.graph.ef_construction = ef;
|
||||
}
|
||||
|
||||
protected:
|
||||
inline const HNSWHeader &header() const {
|
||||
return header_;
|
||||
}
|
||||
|
||||
inline HNSWHeader *mutable_header() {
|
||||
return &header_;
|
||||
}
|
||||
|
||||
inline size_t header_size() const {
|
||||
return sizeof(header_);
|
||||
}
|
||||
|
||||
void set_node_size(size_t size) {
|
||||
header_.graph.node_size = size;
|
||||
}
|
||||
|
||||
//! Dump all segment by dumper
|
||||
//! Return dump size if success, errno(<0) in failure
|
||||
int64_t dump_segments(
|
||||
const IndexDumper::Pointer &dumper, key_t *keys,
|
||||
const std::function<level_t(node_id_t)> &get_level) const;
|
||||
|
||||
private:
|
||||
//! dump mapping segment, for get_vector_by_key in provider
|
||||
int64_t dump_mapping_segment(const IndexDumper::Pointer &dumper,
|
||||
const key_t *keys) const;
|
||||
|
||||
//! dump hnsw head by dumper
|
||||
//! Return dump size if success, errno(<0) in failure
|
||||
int64_t dump_header(const IndexDumper::Pointer &dumper,
|
||||
const HNSWHeader &hd) const;
|
||||
|
||||
//! dump vectors by dumper
|
||||
//! Return dump size if success, errno(<0) in failure
|
||||
int64_t dump_vectors(const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping) const;
|
||||
|
||||
//! dump hnsw neighbors by dumper
|
||||
//! Return dump size if success, errno(<0) in failure
|
||||
int64_t dump_neighbors(const IndexDumper::Pointer &dumper,
|
||||
const std::function<level_t(node_id_t)> &get_level,
|
||||
const std::vector<node_id_t> &reorder_mapping,
|
||||
const std::vector<node_id_t> &neighbor_mapping) const {
|
||||
auto len1 = dump_graph_neighbors(dumper, reorder_mapping, neighbor_mapping);
|
||||
if (len1 < 0) {
|
||||
return len1;
|
||||
}
|
||||
auto len2 = dump_upper_neighbors(dumper, get_level, reorder_mapping,
|
||||
neighbor_mapping);
|
||||
if (len2 < 0) {
|
||||
return len2;
|
||||
}
|
||||
|
||||
return len1 + len2;
|
||||
}
|
||||
|
||||
//! dump segment by dumper
|
||||
//! Return dump size if success, errno(<0) in failure
|
||||
int64_t dump_segment(const IndexDumper::Pointer &dumper,
|
||||
const std::string &segment_id, const void *data,
|
||||
size_t size) const;
|
||||
|
||||
//! Dump level 0 neighbors
|
||||
//! Return dump size if success, errno(<0) in failure
|
||||
int64_t dump_graph_neighbors(
|
||||
const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping,
|
||||
const std::vector<node_id_t> &neighbor_mapping) const;
|
||||
|
||||
//! Dump upper level neighbors
|
||||
//! Return dump size if success, errno(<0) in failure
|
||||
int64_t dump_upper_neighbors(
|
||||
const IndexDumper::Pointer &dumper,
|
||||
const std::function<level_t(node_id_t)> &get_level,
|
||||
const std::vector<node_id_t> &reorder_mapping,
|
||||
const std::vector<node_id_t> &neighbor_mapping) const;
|
||||
|
||||
public:
|
||||
//! Cleanup the entity
|
||||
virtual int cleanup(void) {
|
||||
header_.clear();
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Make a copy of searcher entity, to support thread-safe operation.
|
||||
//! The segment in container cannot be read concurrenly
|
||||
virtual const HnswRabitqEntity::Pointer clone() const {
|
||||
LOG_ERROR("Update neighbors not implemented");
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual key_t get_key(node_id_t id) const = 0;
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector(node_id_t id) const = 0;
|
||||
|
||||
//! Get vectors feature data by keys
|
||||
virtual int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const = 0;
|
||||
|
||||
virtual int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const = 0;
|
||||
virtual int get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const = 0;
|
||||
|
||||
//! Retrieve a vector using a primary key
|
||||
virtual const void *get_vector_by_key(uint64_t /*key*/) const {
|
||||
LOG_ERROR("get vector not implemented");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
virtual int get_vector_by_key(const key_t /*key*/,
|
||||
IndexStorage::MemoryBlock & /*block*/) const {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
//! Note: the neighbors cannot be modified, using the following
|
||||
//! method to get WritableNeighbors if want to
|
||||
virtual const Neighbors get_neighbors(level_t level, node_id_t id) const = 0;
|
||||
|
||||
//! Add vector and key to hnsw entity, and local id will be saved in id
|
||||
virtual int add_vector(level_t /*level*/, key_t /*key*/, const void * /*vec*/,
|
||||
node_id_t * /*id*/) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
//! Add vector and id to hnsw entity
|
||||
virtual int add_vector_with_id(level_t /*level*/, node_id_t /*id*/,
|
||||
const void * /*vec*/) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
virtual int update_neighbors(
|
||||
level_t /*level*/, node_id_t /*id*/,
|
||||
const std::vector<std::pair<node_id_t, ResultRecord>> & /*neighbors*/) {
|
||||
LOG_ERROR("Update neighbors dense not implemented");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Append neighbor_id to node id neighbors on level, size is the current
|
||||
//! neighbors size. Notice: the caller must be ensure the neighbors not full
|
||||
virtual void add_neighbor(level_t /*level*/, node_id_t /*id*/,
|
||||
uint32_t /*size*/, node_id_t /*neighbor_id*/) {
|
||||
LOG_ERROR("Add neighbor not implemented");
|
||||
}
|
||||
|
||||
//! Update entry point and max level
|
||||
virtual void update_ep_and_level(node_id_t ep, level_t level) {
|
||||
header_.hnsw.entry_point = ep;
|
||||
header_.hnsw.max_level = level;
|
||||
}
|
||||
|
||||
virtual int load(const IndexStorage::Pointer & /*container*/,
|
||||
bool /*check_crc*/) {
|
||||
LOG_ERROR("Load not implemented");
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
virtual int dump(const IndexDumper::Pointer & /*dumper*/) {
|
||||
LOG_ERROR("Dump not implemented");
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
static int CalcAndAddPadding(const IndexDumper::Pointer &dumper,
|
||||
size_t data_size, size_t *padding_size);
|
||||
|
||||
uint32_t get_cluster_id(const void *vec) const {
|
||||
return *reinterpret_cast<const uint32_t *>(
|
||||
reinterpret_cast<const char *>(vec) + cluster_id_offset());
|
||||
}
|
||||
|
||||
const char *get_bin_data(const void *vec) const {
|
||||
return reinterpret_cast<const char *>(vec) + bin_data_offset();
|
||||
}
|
||||
|
||||
const char *get_ex_data(const void *vec) const {
|
||||
return reinterpret_cast<const char *>(vec) + ex_data_offset();
|
||||
}
|
||||
|
||||
uint32_t cluster_id_offset() const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t bin_data_offset() const {
|
||||
return cluster_id_offset() + sizeof(uint32_t);
|
||||
}
|
||||
|
||||
uint32_t ex_data_offset() const {
|
||||
return bin_data_offset() + size_bin_data();
|
||||
}
|
||||
|
||||
protected:
|
||||
static inline size_t AlignSize(size_t size) {
|
||||
return (size + 0x1F) & (~0x1F);
|
||||
}
|
||||
|
||||
static inline size_t AlignPageSize(size_t size) {
|
||||
size_t page_mask = ailego::MemoryHelper::PageSize() - 1;
|
||||
return (size + page_mask) & (~page_mask);
|
||||
}
|
||||
|
||||
static inline size_t AlignHugePageSize(size_t size) {
|
||||
size_t page_mask = ailego::MemoryHelper::HugePageSize() - 1;
|
||||
return (size + page_mask) & (~page_mask);
|
||||
}
|
||||
|
||||
//! rearrange vectors to improve cache locality
|
||||
void reshuffle_vectors(const std::function<level_t(node_id_t)> &get_level,
|
||||
std::vector<node_id_t> *n2o_mapping,
|
||||
std::vector<node_id_t> *o2n_mapping,
|
||||
key_t *keys) const;
|
||||
|
||||
public:
|
||||
const static std::string kGraphHeaderSegmentId;
|
||||
const static std::string kGraphFeaturesSegmentId;
|
||||
const static std::string kGraphKeysSegmentId;
|
||||
const static std::string kGraphNeighborsSegmentId;
|
||||
const static std::string kGraphOffsetsSegmentId;
|
||||
const static std::string kGraphMappingSegmentId;
|
||||
const static std::string kHnswHeaderSegmentId;
|
||||
const static std::string kHnswNeighborsSegmentId;
|
||||
const static std::string kHnswOffsetsSegmentId;
|
||||
|
||||
constexpr static uint32_t kRevision = 0U;
|
||||
constexpr static size_t kMaxGraphLayers = 15;
|
||||
constexpr static uint32_t kDefaultEfConstruction = 500;
|
||||
constexpr static uint32_t kDefaultEf = 500;
|
||||
constexpr static uint32_t kDefaultUpperMaxNeighborCnt = 50; // M of HNSW
|
||||
constexpr static uint32_t kDefaultL0MaxNeighborCnt = 100;
|
||||
constexpr static uint32_t kMaxNeighborCnt = 65535;
|
||||
constexpr static float kDefaultScanRatio = 0.1f;
|
||||
constexpr static uint32_t kDefaultMinScanLimit = 10000;
|
||||
constexpr static uint32_t kDefaultMaxScanLimit =
|
||||
std::numeric_limits<uint32_t>::max();
|
||||
constexpr static float kDefaultBFNegativeProbability = 0.001f;
|
||||
constexpr static uint32_t kDefaultScalingFactor = 50U;
|
||||
constexpr static uint32_t kDefaultBruteForceThreshold = 1000U;
|
||||
constexpr static uint32_t kDefaultDocsHardLimit = 1 << 30U; // 1 billion
|
||||
constexpr static float kDefaultDocsSoftLimitRatio = 0.9f;
|
||||
constexpr static size_t kMaxChunkSize = 0xFFFFFFFF;
|
||||
constexpr static size_t kDefaultChunkSize = 2UL * 1024UL * 1024UL;
|
||||
constexpr static size_t kDefaultMaxChunkCnt = 50000UL;
|
||||
constexpr static float kDefaultNeighborPruneMultiplier =
|
||||
1.0f; // prune_cnt = upper_max_neighbor_cnt * multiplier
|
||||
constexpr static float kDefaultL0MaxNeighborCntMultiplier =
|
||||
2.0f; // l0_max_neighbor_cnt = upper_max_neighbor_cnt * multiplier
|
||||
|
||||
protected:
|
||||
HNSWHeader header_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,231 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "hnsw_rabitq_chunk.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! Persistent hashmap implement through open addressing algorithm
|
||||
template <class Key, class Val, Val EmptyVal = 0U,
|
||||
typename =
|
||||
typename std::enable_if<std::is_integral<Key>::value>::type>
|
||||
class HnswIndexHashMap {
|
||||
using key_type = Key;
|
||||
using val_type = Val;
|
||||
|
||||
struct Iterator {
|
||||
key_type first;
|
||||
val_type second;
|
||||
};
|
||||
typedef Iterator *iterator;
|
||||
typedef Iterator Item;
|
||||
typedef const Iterator *const_iterator;
|
||||
|
||||
class Slot {
|
||||
public:
|
||||
Slot(Chunk::Pointer &&chunk, const void *data)
|
||||
: chunk_(std::move(chunk)),
|
||||
items_(reinterpret_cast<const Item *>(data)) {}
|
||||
//! Return a empty loc or the key item loc
|
||||
|
||||
Slot(Chunk::Pointer &&chunk, IndexStorage::MemoryBlock &&mem_block)
|
||||
: chunk_(std::move(chunk)), items_block_(std::move(mem_block)) {
|
||||
items_ = reinterpret_cast<const Item *>(items_block_.data());
|
||||
}
|
||||
const_iterator find(key_type key, uint32_t max_items, uint32_t mask) const {
|
||||
auto it = &items_[key & mask];
|
||||
for (auto i = 0U; i < max_items; ++i) {
|
||||
if (it->first == key || it->second == EmptyVal) {
|
||||
// LOG_DEBUG("i=%u", i);
|
||||
return it;
|
||||
}
|
||||
++it;
|
||||
if (it == &items_[max_items]) {
|
||||
it = &items_[0];
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool update(const_iterator it) {
|
||||
uint32_t offset = reinterpret_cast<const uint8_t *>(it) -
|
||||
reinterpret_cast<const uint8_t *>(&items_[0]);
|
||||
if (ailego_unlikely(chunk_->write(offset, it, sizeof(Item)) !=
|
||||
sizeof(Item))) {
|
||||
LOG_ERROR("Chunk write failed");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
Chunk::Pointer chunk_{};
|
||||
const Item *items_{nullptr}; // point to chunk data
|
||||
IndexStorage::MemoryBlock items_block_{};
|
||||
};
|
||||
|
||||
public:
|
||||
//! Init the hash
|
||||
//! broker the index allocator
|
||||
//! chunk_size the size of per chunk allocated, actual size may greater
|
||||
//! factor factor = 1/ratio, ratio is the probability of a squence
|
||||
//! number inserted to this container
|
||||
//! max the max number key can be inserted
|
||||
//! expansion_ratio memory expansion ratio
|
||||
int init(HnswRabitqChunkBroker::Pointer &broker, uint32_t chunk_size,
|
||||
uint32_t factor, size_t max, float expansion_ratio) {
|
||||
ailego_assert_with(expansion_ratio > 1.0f, "ratio must > 1.0f");
|
||||
broker_ = broker;
|
||||
|
||||
size_t items = std::ceil(chunk_size * 1.0f / sizeof(Item));
|
||||
slot_items_ = 1UL << static_cast<size_t>((std::ceil(std::log2(items))));
|
||||
size_t range = slot_items_ * factor / expansion_ratio;
|
||||
mask_bits_ = std::floor(std::log2(range));
|
||||
range = 1UL << mask_bits_;
|
||||
size_t max_slots = std::ceil(max * 1.0f / range);
|
||||
slots_.reserve(max_slots);
|
||||
slot_loc_mask_ = slot_items_ - 1U;
|
||||
int ret = load();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_DEBUG(
|
||||
"HnswRabitqIndexHash init, chunkSize=%u factor=%u max=%zu "
|
||||
"ratio=%f slotItems=%u maxSlots=%zu maskBits=%u "
|
||||
"range=%zu",
|
||||
chunk_size, factor, max, expansion_ratio, slot_items_, max_slots,
|
||||
mask_bits_, range);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cleanup(void) {
|
||||
broker_.reset();
|
||||
slots_.clear();
|
||||
slots_.shrink_to_fit();
|
||||
mask_bits_ = 0U;
|
||||
slot_items_ = 0U;
|
||||
slot_loc_mask_ = 0U;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const_iterator end(void) const {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const_iterator find(const key_type key) const {
|
||||
auto idx = key >> mask_bits_;
|
||||
if (idx >= slots_.size()) {
|
||||
return end();
|
||||
}
|
||||
auto it = slots_[idx].find(key, slot_items_, slot_loc_mask_);
|
||||
return it && it->second != EmptyVal ? it : nullptr;
|
||||
}
|
||||
|
||||
bool insert(key_type key, val_type val) {
|
||||
auto idx = key >> mask_bits_;
|
||||
if (idx >= slots_.size()) {
|
||||
if (ailego_unlikely(idx >= slots_.capacity())) {
|
||||
LOG_ERROR("no space to insert");
|
||||
return false;
|
||||
}
|
||||
for (auto i = slots_.size(); i <= idx; ++i) {
|
||||
if (ailego_unlikely(!alloc_slot(i))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
auto it = slots_[idx].find(key, slot_items_, slot_loc_mask_);
|
||||
if (ailego_unlikely(it == nullptr)) {
|
||||
LOG_ERROR("no space to insert");
|
||||
return false;
|
||||
}
|
||||
|
||||
//! TODO: write memory is ok?
|
||||
const_cast<iterator>(it)->first = key;
|
||||
const_cast<iterator>(it)->second = val;
|
||||
|
||||
return slots_[idx].update(it);
|
||||
}
|
||||
|
||||
private:
|
||||
bool alloc_slot(size_t idx) {
|
||||
ailego_assert_with(idx == slots_.size(), "invalid idx");
|
||||
|
||||
size_t size = slot_items_ * sizeof(Item);
|
||||
auto p = broker_->alloc_chunk(
|
||||
HnswRabitqChunkBroker::CHUNK_TYPE_NEIGHBOR_INDEX, idx, size);
|
||||
if (ailego_unlikely(p.first != 0)) {
|
||||
LOG_ERROR("Alloc data chunk failed");
|
||||
return false;
|
||||
}
|
||||
Chunk::Pointer chunk = p.second;
|
||||
if (ailego_unlikely(chunk->resize(size) != size)) {
|
||||
LOG_ERROR("Chunk resize failed, size=%zu", size);
|
||||
return false;
|
||||
}
|
||||
//! Read the whole data to memory
|
||||
IndexStorage::MemoryBlock data_block;
|
||||
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
|
||||
LOG_ERROR("Chunk read failed, size=%zu", size);
|
||||
return false;
|
||||
}
|
||||
|
||||
slots_.emplace_back(std::move(chunk), std::move(data_block));
|
||||
return true;
|
||||
}
|
||||
|
||||
int load(void) {
|
||||
size_t slots_cnt = broker_->get_chunk_cnt(
|
||||
HnswRabitqChunkBroker::CHUNK_TYPE_NEIGHBOR_INDEX);
|
||||
for (size_t i = 0UL; i < slots_cnt; ++i) {
|
||||
auto chunk = broker_->get_chunk(
|
||||
HnswRabitqChunkBroker::CHUNK_TYPE_NEIGHBOR_INDEX, i);
|
||||
if (!chunk) {
|
||||
LOG_ERROR("Get chunk failed, seq=%zu", i);
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
size_t size = sizeof(Item) * slot_items_;
|
||||
if (chunk->data_size() < size) {
|
||||
LOG_ERROR(
|
||||
"Hash params may be mismatch, seq=%zu, data_size=%zu "
|
||||
"expect=%zu",
|
||||
i, chunk->data_size(), size);
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
//! Read the whole data to memory
|
||||
IndexStorage::MemoryBlock data_block;
|
||||
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
|
||||
LOG_ERROR("Chunk read failed, size=%zu", size);
|
||||
return false;
|
||||
}
|
||||
slots_.emplace_back(std::move(chunk), std::move(data_block));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
HnswRabitqChunkBroker::Pointer broker_{}; // chunk broker
|
||||
std::vector<Slot> slots_{};
|
||||
uint32_t mask_bits_{0U};
|
||||
uint32_t slot_items_{}; // must be a power of 2
|
||||
uint32_t slot_loc_mask_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,134 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "zvec/core/framework/index_provider.h"
|
||||
#include "zvec/core/framework/index_searcher.h"
|
||||
#include "zvec/core/framework/index_streamer.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class HnswRabitqIndexProvider : public IndexProvider {
|
||||
public:
|
||||
HnswRabitqIndexProvider(const IndexMeta &meta,
|
||||
const HnswRabitqEntity::Pointer &entity,
|
||||
const std::string &owner)
|
||||
: meta_(meta), entity_(entity), owner_class_(owner) {}
|
||||
|
||||
HnswRabitqIndexProvider(const HnswRabitqIndexProvider &) = delete;
|
||||
HnswRabitqIndexProvider &operator=(const HnswRabitqIndexProvider &) = delete;
|
||||
|
||||
public: // holder interface
|
||||
//! Create a new iterator
|
||||
IndexProvider::Iterator::Pointer create_iterator() override {
|
||||
return HnswRabitqIndexProvider::Iterator::Pointer(new (std::nothrow)
|
||||
Iterator(entity_));
|
||||
}
|
||||
|
||||
//! Retrieve count of vectors
|
||||
size_t count(void) const override {
|
||||
return entity_->doc_cnt();
|
||||
}
|
||||
|
||||
//! Retrieve dimension of vector
|
||||
size_t dimension(void) const override {
|
||||
return meta_.dimension();
|
||||
}
|
||||
|
||||
//! Retrieve type of vector
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return meta_.data_type();
|
||||
}
|
||||
|
||||
//! Retrieve vector size in bytes
|
||||
size_t element_size(void) const override {
|
||||
return meta_.element_size();
|
||||
}
|
||||
|
||||
public: // provider's unique interface
|
||||
//! Retrieve a vector using a primary key
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
return entity_->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
//! Retrieve the owner class
|
||||
const std::string &owner_class(void) const override {
|
||||
return owner_class_;
|
||||
}
|
||||
|
||||
private:
|
||||
class Iterator : public IndexProvider::Iterator {
|
||||
public:
|
||||
Iterator(const HnswRabitqEntity::Pointer &entity)
|
||||
: entity_(entity), cur_id_(0U) {}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
//! NOTICE: the vec feature will be changed after iterating to next, so
|
||||
//! the caller need to keep a copy of it before iterator to next vector
|
||||
virtual const void *data(void) const override {
|
||||
return entity_->get_vector(cur_id_);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
return cur_id_ < entity_->doc_cnt();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
return entity_->get_key(cur_id_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
// cur_id_ += 1;
|
||||
cur_id_ = get_next_valid_id(cur_id_ + 1);
|
||||
}
|
||||
|
||||
//! Reset the iterator
|
||||
void reset(void) {
|
||||
cur_id_ = get_next_valid_id(0);
|
||||
}
|
||||
|
||||
private:
|
||||
node_id_t get_next_valid_id(node_id_t start_id) {
|
||||
for (node_id_t i = start_id; i < entity_->doc_cnt(); i++) {
|
||||
if (entity_->get_key(i) != kInvalidNodeId) {
|
||||
cur_id_ = i;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return kInvalidNodeId;
|
||||
}
|
||||
|
||||
private:
|
||||
const HnswRabitqEntity::Pointer entity_;
|
||||
node_id_t cur_id_;
|
||||
};
|
||||
|
||||
private:
|
||||
const IndexMeta &meta_;
|
||||
const HnswRabitqEntity::Pointer entity_;
|
||||
const std::string owner_class_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,121 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
inline const std::string PARAM_HNSW_RABITQ_GENERAL_DIMENSION(
|
||||
"proxima.hnsw_rabitq.general.dimension");
|
||||
|
||||
inline const std::string PARAM_HNSW_RABITQ_BUILDER_THREAD_COUNT(
|
||||
"proxima.hnsw_rabitq.builder.thread_count");
|
||||
inline const std::string PARAM_HNSW_RABITQ_BUILDER_MEMORY_QUOTA(
|
||||
"proxima.hnsw_rabitq.builder.memory_quota");
|
||||
inline const std::string PARAM_HNSW_RABITQ_BUILDER_EFCONSTRUCTION(
|
||||
"proxima.hnsw_rabitq.builder.efconstruction");
|
||||
inline const std::string PARAM_HNSW_RABITQ_BUILDER_SCALING_FACTOR(
|
||||
"proxima.hnsw_rabitq.builder.scaling_factor");
|
||||
inline const std::string PARAM_HNSW_RABITQ_BUILDER_CHECK_INTERVAL_SECS(
|
||||
"proxima.hnsw_rabitq.builder.check_interval_secs");
|
||||
inline const std::string PARAM_HNSW_RABITQ_BUILDER_NEIGHBOR_PRUNE_MULTIPLIER(
|
||||
"proxima.hnsw_rabitq.builder.neighbor_prune_multiplier");
|
||||
inline const std::string PARAM_HNSW_RABITQ_BUILDER_MIN_NEIGHBOR_COUNT(
|
||||
"proxima.hnsw_rabitq.builder.min_neighbor_count");
|
||||
inline const std::string PARAM_HNSW_RABITQ_BUILDER_MAX_NEIGHBOR_COUNT(
|
||||
"proxima.hnsw_rabitq.builder.max_neighbor_count");
|
||||
inline const std::string
|
||||
PARAM_HNSW_RABITQ_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
|
||||
"proxima.hnsw_rabitq.builder.l0_max_neighbor_count_multiplier");
|
||||
|
||||
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_EF(
|
||||
"proxima.hnsw_rabitq.searcher.ef");
|
||||
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD(
|
||||
"proxima.hnsw_rabitq.searcher.brute_force_threshold");
|
||||
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_NEIGHBORS_IN_MEMORY_ENABLE(
|
||||
"proxima.hnsw_rabitq.searcher.neighbors_in_memory_enable");
|
||||
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO(
|
||||
"proxima.hnsw_rabitq.searcher.max_scan_ratio");
|
||||
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_CHECK_CRC_ENABLE(
|
||||
"proxima.hnsw_rabitq.searcher.check_crc_enable");
|
||||
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_ENABLE(
|
||||
"proxima.hnsw_rabitq.searcher.visit_bloomfilter_enable");
|
||||
inline const std::string
|
||||
PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB(
|
||||
"proxima.hnsw_rabitq.searcher.visit_bloomfilter_negative_prob");
|
||||
inline const std::string PARAM_HNSW_RABITQ_SEARCHER_FORCE_PADDING_RESULT_ENABLE(
|
||||
"proxima.hnsw_rabitq.searcher.force_padding_result_enable");
|
||||
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO(
|
||||
"proxima.hnsw_rabitq.streamer.max_scan_ratio");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT(
|
||||
"proxima.hnsw_rabitq.streamer.min_scan_limit");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT(
|
||||
"proxima.hnsw_rabitq.streamer.max_scan_limit");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_EF(
|
||||
"proxima.hnsw_rabitq.streamer.ef");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_EFCONSTRUCTION(
|
||||
"proxima.hnsw_rabitq.streamer.efconstruction");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT(
|
||||
"proxima.hnsw_rabitq.streamer.max_neighbor_count");
|
||||
inline const std::string
|
||||
PARAM_HNSW_RABITQ_STREAMER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
|
||||
"proxima.hnsw_rabitq.streamer.l0_max_neighbor_count_multiplier");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_SCALING_FACTOR(
|
||||
"proxima.hnsw_rabitq.streamer.scaling_factor");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD(
|
||||
"proxima.hnsw_rabitq.streamer.brute_force_threshold");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT(
|
||||
"proxima.hnsw_rabitq.streamer.docs_hard_limit");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT(
|
||||
"proxima.hnsw_rabitq.streamer.docs_soft_limit");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MAX_INDEX_SIZE(
|
||||
"proxima.hnsw_rabitq.streamer.max_index_size");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_ENABLE(
|
||||
"proxima.hnsw_rabitq.streamer.visit_bloomfilter_enable");
|
||||
inline const std::string
|
||||
PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB(
|
||||
"proxima.hnsw_rabitq.streamer.visit_bloomfilter_negative_prob");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_CHECK_CRC_ENABLE(
|
||||
"proxima.hnsw_rabitq.streamer.check_crc_enable");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_NEIGHBOR_PRUNE_MULTIPLIER(
|
||||
"proxima.hnsw_rabitq.streamer.neighbor_prune_multiplier");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_CHUNK_SIZE(
|
||||
"proxima.hnsw_rabitq.streamer.chunk_size");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_FILTER_SAME_KEY(
|
||||
"proxima.hnsw_rabitq.streamer.filter_same_key");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_GET_VECTOR_ENABLE(
|
||||
"proxima.hnsw_rabitq.streamer.get_vector_enable");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_MIN_NEIGHBOR_COUNT(
|
||||
"proxima.hnsw_rabitq.streamer.min_neighbor_count");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_FORCE_PADDING_RESULT_ENABLE(
|
||||
"proxima.hnsw_rabitq.streamer.force_padding_result_enable");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_ESTIMATE_DOC_COUNT(
|
||||
"proxima.hnsw_rabitq.streamer.estimate_doc_count");
|
||||
inline const std::string PARAM_HNSW_RABITQ_STREAMER_USE_ID_MAP(
|
||||
"proxima.hnsw_rabitq.streamer.use_id_map");
|
||||
|
||||
inline const std::string PARAM_HNSW_RABITQ_REDUCER_WORKING_PATH(
|
||||
"proxima.hnsw_rabitq.reducer.working_path");
|
||||
inline const std::string PARAM_HNSW_RABITQ_REDUCER_NUM_OF_ADD_THREADS(
|
||||
"proxima.hnsw_rabitq.reducer.num_of_add_threads");
|
||||
inline const std::string PARAM_HNSW_RABITQ_REDUCER_INDEX_NAME(
|
||||
"proxima.hnsw_rabitq.reducer.index_name");
|
||||
inline const std::string PARAM_HNSW_RABITQ_REDUCER_EFCONSTRUCTION(
|
||||
"proxima.hnsw_rabitq.reducer.efconstruction");
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,354 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_query_algorithm.h"
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include <rabitqlib/index/estimator.hpp>
|
||||
#include "zvec/ailego/internal/platform.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
#include "hnsw_rabitq_query_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswRabitqQueryAlgorithm::HnswRabitqQueryAlgorithm(HnswRabitqEntity &entity,
|
||||
size_t num_clusters,
|
||||
RabitqMetricType metric_type)
|
||||
: entity_(entity),
|
||||
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
|
||||
lock_pool_(kLockCnt),
|
||||
num_clusters_(num_clusters),
|
||||
metric_type_(metric_type) {
|
||||
ex_bits_ = entity_.ex_bits();
|
||||
padded_dim_ = entity_.padded_dim();
|
||||
ip_func_ = rabitqlib::select_excode_ipfunc(ex_bits_);
|
||||
LOG_INFO(
|
||||
"Create query algorithm. num_clusters=%zu ex_bits=%zu padded_dim=%zu",
|
||||
num_clusters_, ex_bits_, padded_dim_);
|
||||
}
|
||||
|
||||
int HnswRabitqQueryAlgorithm::cleanup() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqQueryAlgorithm::search(HnswRabitqQueryEntity *entity,
|
||||
HnswRabitqContext *ctx) const {
|
||||
spin_lock_.lock();
|
||||
auto maxLevel = entity_.cur_max_level();
|
||||
auto entry_point = entity_.entry_point();
|
||||
spin_lock_.unlock();
|
||||
|
||||
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
EstimateRecord curest;
|
||||
get_bin_est(entity_.get_vector(entry_point), curest, *entity);
|
||||
|
||||
for (level_t cur_level = maxLevel; cur_level >= 1; --cur_level) {
|
||||
select_entry_point(cur_level, &entry_point, &curest, ctx, entity);
|
||||
}
|
||||
|
||||
auto &topk_heap = ctx->topk_heap();
|
||||
topk_heap.clear();
|
||||
search_neighbors(0, &entry_point, &curest, topk_heap, ctx, entity);
|
||||
|
||||
if (ctx->group_by_search()) {
|
||||
expand_neighbors_by_group(topk_heap, ctx, entity);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//! select_entry_point on hnsw level, ef = 1
|
||||
void HnswRabitqQueryAlgorithm::select_entry_point(
|
||||
level_t level, node_id_t *entry_point, EstimateRecord *curest,
|
||||
HnswRabitqContext *ctx, HnswRabitqQueryEntity *query_entity) const {
|
||||
auto &entity = ctx->get_entity();
|
||||
while (true) {
|
||||
const Neighbors neighbors = entity.get_neighbors(level, *entry_point);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
ailego_prefetch(neighbors.data);
|
||||
uint32_t size = neighbors.size();
|
||||
if (size == 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
bool find_closer = false;
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
EstimateRecord candest;
|
||||
get_bin_est(entity_.get_vector(neighbors[i]), candest, *query_entity);
|
||||
|
||||
if (candest.est_dist < curest->est_dist) {
|
||||
*curest = candest;
|
||||
*entry_point = neighbors[i];
|
||||
find_closer = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!find_closer) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswRabitqQueryAlgorithm::search_neighbors(
|
||||
level_t level, node_id_t *entry_point, EstimateRecord *dist, TopkHeap &topk,
|
||||
HnswRabitqContext *ctx, HnswRabitqQueryEntity *query_entity) const {
|
||||
const auto &entity = ctx->get_entity();
|
||||
VisitFilter &visit = ctx->visit_filter();
|
||||
CandidateHeap &candidates = ctx->candidates();
|
||||
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
|
||||
if (ctx->filter().is_valid()) {
|
||||
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
|
||||
}
|
||||
|
||||
candidates.clear();
|
||||
visit.clear();
|
||||
visit.set_visited(*entry_point);
|
||||
if (!filter(*entry_point)) {
|
||||
topk.emplace(*entry_point, ResultRecord(*dist));
|
||||
}
|
||||
|
||||
candidates.emplace(*entry_point, ResultRecord(*dist));
|
||||
while (!candidates.empty() && !ctx->reach_scan_limit()) {
|
||||
auto top = candidates.begin();
|
||||
node_id_t main_node = top->first;
|
||||
auto main_dist = top->second;
|
||||
|
||||
if (topk.full() && main_dist.est_dist > topk[0].second.est_dist) {
|
||||
break;
|
||||
}
|
||||
|
||||
candidates.pop();
|
||||
const Neighbors neighbors = entity.get_neighbors(level, main_node);
|
||||
ailego_prefetch(neighbors.data);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
|
||||
std::vector<node_id_t> neighbor_ids(neighbors.size());
|
||||
uint32_t size = 0;
|
||||
for (uint32_t i = 0; i < neighbors.size(); ++i) {
|
||||
node_id_t node = neighbors[i];
|
||||
if (visit.visited(node)) {
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_visit_dup_cnt())++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
visit.set_visited(node);
|
||||
neighbor_ids[size++] = node;
|
||||
}
|
||||
if (size == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
node_id_t node = neighbor_ids[i];
|
||||
EstimateRecord candest;
|
||||
auto *cand_vector = entity_.get_vector(node);
|
||||
ailego_prefetch(cand_vector);
|
||||
get_bin_est(cand_vector, candest, *query_entity);
|
||||
|
||||
if (ex_bits_ > 0) {
|
||||
// Check preliminary score against current worst full estimate.
|
||||
bool flag_update_KNNs =
|
||||
(!topk.full()) || candest.low_dist < topk[0].second.est_dist;
|
||||
|
||||
if (flag_update_KNNs) {
|
||||
// Compute the full estimate if promising.
|
||||
get_full_est(cand_vector, candest, *query_entity);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
// ex_bits_ == 0: est_dist is already the best estimate
|
||||
if (topk.full() && candest.est_dist >= topk[0].second.est_dist) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
candidates.emplace(node, ResultRecord(candest));
|
||||
// update entry_point for next level scan
|
||||
if (candest < *dist) {
|
||||
*entry_point = node;
|
||||
*dist = candest;
|
||||
}
|
||||
if (!filter(node)) {
|
||||
topk.emplace(node, ResultRecord(candest));
|
||||
}
|
||||
} // end for
|
||||
} // while
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswRabitqQueryAlgorithm::expand_neighbors_by_group(
|
||||
TopkHeap &topk, HnswRabitqContext *ctx,
|
||||
HnswRabitqQueryEntity *query_entity) const {
|
||||
if (!ctx->group_by().is_valid()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto &entity = ctx->get_entity();
|
||||
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
|
||||
return ctx->group_by()(entity.get_key(id));
|
||||
};
|
||||
|
||||
// devide into groups
|
||||
std::map<std::string, TopkHeap> &group_topk_heaps = ctx->group_topk_heaps();
|
||||
for (uint32_t i = 0; i < topk.size(); ++i) {
|
||||
node_id_t id = topk[i].first;
|
||||
auto score = topk[i].second;
|
||||
|
||||
std::string group_id = group_by(id);
|
||||
|
||||
auto &topk_heap = group_topk_heaps[group_id];
|
||||
if (topk_heap.empty()) {
|
||||
topk_heap.limit(ctx->group_topk());
|
||||
}
|
||||
topk_heap.emplace_back(id, score);
|
||||
}
|
||||
|
||||
// stage 2, expand to reach group num as possible
|
||||
if (group_topk_heaps.size() < ctx->group_num()) {
|
||||
VisitFilter &visit = ctx->visit_filter();
|
||||
CandidateHeap &candidates = ctx->candidates();
|
||||
|
||||
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
|
||||
if (ctx->filter().is_valid()) {
|
||||
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
|
||||
}
|
||||
|
||||
// refill to get enough groups
|
||||
candidates.clear();
|
||||
visit.clear();
|
||||
for (uint32_t i = 0; i < topk.size(); ++i) {
|
||||
node_id_t id = topk[i].first;
|
||||
auto score = topk[i].second;
|
||||
|
||||
visit.set_visited(id);
|
||||
candidates.emplace_back(id, score);
|
||||
}
|
||||
|
||||
// do expand
|
||||
while (!candidates.empty() && !ctx->reach_scan_limit()) {
|
||||
auto top = candidates.begin();
|
||||
node_id_t main_node = top->first;
|
||||
|
||||
candidates.pop();
|
||||
const Neighbors neighbors = entity.get_neighbors(0, main_node);
|
||||
ailego_prefetch(neighbors.data);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
|
||||
std::vector<node_id_t> neighbor_ids(neighbors.size());
|
||||
uint32_t size = 0;
|
||||
for (uint32_t i = 0; i < neighbors.size(); ++i) {
|
||||
node_id_t node = neighbors[i];
|
||||
if (visit.visited(node)) {
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_visit_dup_cnt())++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
visit.set_visited(node);
|
||||
neighbor_ids[size++] = node;
|
||||
}
|
||||
if (size == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
node_id_t node = neighbor_ids[i];
|
||||
EstimateRecord candest;
|
||||
auto *cand_vector = entity_.get_vector(node);
|
||||
ailego_prefetch(cand_vector);
|
||||
get_full_est(cand_vector, candest, *query_entity);
|
||||
|
||||
if (!filter(node)) {
|
||||
std::string group_id = group_by(node);
|
||||
|
||||
auto &topk_heap = group_topk_heaps[group_id];
|
||||
if (topk_heap.empty()) {
|
||||
topk_heap.limit(ctx->group_topk());
|
||||
}
|
||||
topk_heap.emplace_back(node, ResultRecord(candest));
|
||||
|
||||
if (group_topk_heaps.size() >= ctx->group_num()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
candidates.emplace(node, ResultRecord(candest));
|
||||
} // end for
|
||||
} // end while
|
||||
} // end if
|
||||
}
|
||||
|
||||
void HnswRabitqQueryAlgorithm::get_bin_est(
|
||||
const void *vector, EstimateRecord &res,
|
||||
HnswRabitqQueryEntity &entity) const {
|
||||
const auto &q_to_centroids = entity.q_to_centroids;
|
||||
auto &query_wrapper = *entity.query_wrapper;
|
||||
uint32_t cluster_id = entity_.get_cluster_id(vector);
|
||||
const char *bin_data = entity_.get_bin_data(vector);
|
||||
if (metric_type_ == RabitqMetricType::kIP) {
|
||||
float norm = q_to_centroids[cluster_id];
|
||||
float error = q_to_centroids[cluster_id + num_clusters_];
|
||||
rabitqlib::split_single_estdist(bin_data, query_wrapper, padded_dim_,
|
||||
res.ip_x0_qr, res.est_dist, res.low_dist,
|
||||
-norm, error);
|
||||
} else {
|
||||
// L2 distance
|
||||
float norm = q_to_centroids[cluster_id];
|
||||
rabitqlib::split_single_estdist(bin_data, query_wrapper, padded_dim_,
|
||||
res.ip_x0_qr, res.est_dist, res.low_dist,
|
||||
norm * norm, norm);
|
||||
}
|
||||
}
|
||||
|
||||
void HnswRabitqQueryAlgorithm::get_full_est(
|
||||
const void *vector, EstimateRecord &res,
|
||||
HnswRabitqQueryEntity &entity) const {
|
||||
const auto &q_to_centroids = entity.q_to_centroids;
|
||||
auto &query_wrapper = *entity.query_wrapper;
|
||||
uint32_t cluster_id = entity_.get_cluster_id(vector);
|
||||
const char *bin_data = entity_.get_bin_data(vector);
|
||||
const char *ex_data = entity_.get_ex_data(vector);
|
||||
|
||||
if (metric_type_ == RabitqMetricType::kIP) {
|
||||
float norm = q_to_centroids[cluster_id];
|
||||
float error = q_to_centroids[cluster_id + num_clusters_];
|
||||
rabitqlib::split_single_fulldist(bin_data, ex_data, ip_func_, query_wrapper,
|
||||
padded_dim_, ex_bits_, res.est_dist,
|
||||
res.low_dist, res.ip_x0_qr, -norm, error);
|
||||
} else {
|
||||
// L2 distance
|
||||
float norm = q_to_centroids[cluster_id];
|
||||
rabitqlib::split_single_fulldist(
|
||||
bin_data, ex_data, ip_func_, query_wrapper, padded_dim_, ex_bits_,
|
||||
res.est_dist, res.low_dist, res.ip_x0_qr, norm * norm, norm);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,136 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include "hnsw_rabitq_context.h"
|
||||
#include "hnsw_rabitq_dist_calculator.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
#include "rabitq_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class HnswRabitqQueryEntity;
|
||||
|
||||
//! hnsw graph algorithm implement
|
||||
class HnswRabitqQueryAlgorithm {
|
||||
public:
|
||||
typedef std::unique_ptr<HnswRabitqQueryAlgorithm> UPointer;
|
||||
|
||||
public:
|
||||
//! Constructor
|
||||
explicit HnswRabitqQueryAlgorithm(HnswRabitqEntity &entity,
|
||||
size_t num_clusters,
|
||||
RabitqMetricType metric_type);
|
||||
|
||||
//! Destructor
|
||||
~HnswRabitqQueryAlgorithm() = default;
|
||||
|
||||
//! Cleanup HnswRabitqQueryAlgorithm
|
||||
int cleanup();
|
||||
|
||||
//! do knn search in graph
|
||||
//! return 0 on success, or errCode in failure. results saved in ctx
|
||||
int search(HnswRabitqQueryEntity *entity, HnswRabitqContext *ctx) const;
|
||||
|
||||
//! Initiate HnswRabitqQueryAlgorithm
|
||||
int init() {
|
||||
level_probas_.clear();
|
||||
double level_mult =
|
||||
1 / std::log(static_cast<double>(entity_.scaling_factor()));
|
||||
for (int level = 0;; level++) {
|
||||
// refers faiss get_random_level alg
|
||||
double proba =
|
||||
std::exp(-level / level_mult) * (1 - std::exp(-1 / level_mult));
|
||||
if (proba < 1e-9) {
|
||||
break;
|
||||
}
|
||||
level_probas_.push_back(proba);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Generate a random level
|
||||
//! return graph level
|
||||
uint32_t get_random_level() const {
|
||||
// gen rand float (0, 1)
|
||||
double f = mt_() / static_cast<float>(mt_.max());
|
||||
for (size_t level = 0; level < level_probas_.size(); level++) {
|
||||
if (f < level_probas_[level]) {
|
||||
return level;
|
||||
}
|
||||
f -= level_probas_[level];
|
||||
}
|
||||
return level_probas_.size() - 1;
|
||||
}
|
||||
void get_full_est(node_id_t id, EstimateRecord &res,
|
||||
HnswRabitqQueryEntity &entity) const {
|
||||
return get_full_est(entity_.get_vector(id), res, entity);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Select in upper layer to get entry point for next layer search
|
||||
void select_entry_point(level_t level, node_id_t *entry_point,
|
||||
EstimateRecord *dist, HnswRabitqContext *ctx,
|
||||
HnswRabitqQueryEntity *entity) const;
|
||||
|
||||
|
||||
//! Given a node id and level, search the nearest neighbors in graph
|
||||
//! Note: the nearest neighbors result keeps in topk, and entry_point and
|
||||
//! dist will be updated to current level nearest node id and distance
|
||||
void search_neighbors(level_t level, node_id_t *entry_point,
|
||||
EstimateRecord *dist, TopkHeap &topk,
|
||||
HnswRabitqContext *ctx,
|
||||
HnswRabitqQueryEntity *entity) const;
|
||||
|
||||
|
||||
//! expand neighbors until group nums are reached
|
||||
void expand_neighbors_by_group(TopkHeap &topk, HnswRabitqContext *ctx,
|
||||
HnswRabitqQueryEntity *query_entity) const;
|
||||
|
||||
void get_full_est(const void *vector, EstimateRecord &res,
|
||||
HnswRabitqQueryEntity &entity) const;
|
||||
void get_bin_est(const void *vector, EstimateRecord &res,
|
||||
HnswRabitqQueryEntity &entity) const;
|
||||
|
||||
private:
|
||||
HnswRabitqQueryAlgorithm(const HnswRabitqQueryAlgorithm &) = delete;
|
||||
HnswRabitqQueryAlgorithm &operator=(const HnswRabitqQueryAlgorithm &) =
|
||||
delete;
|
||||
|
||||
|
||||
private:
|
||||
static constexpr uint32_t kLockCnt{1U << 8};
|
||||
static constexpr uint32_t kLockMask{kLockCnt - 1U};
|
||||
|
||||
HnswRabitqEntity &entity_;
|
||||
mutable std::mt19937 mt_{};
|
||||
std::vector<double> level_probas_{};
|
||||
|
||||
mutable ailego::SpinMutex spin_lock_{}; // global spin lock
|
||||
std::mutex mutex_{}; // global mutex
|
||||
// TODO: spin lock?
|
||||
std::vector<std::mutex> lock_pool_{};
|
||||
size_t num_clusters_{0};
|
||||
RabitqMetricType metric_type_{RabitqMetricType::kL2};
|
||||
size_t padded_dim_{0};
|
||||
size_t ex_bits_{0};
|
||||
float (*ip_func_)(const float *, const uint8_t *, size_t);
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2025-present the centaurdb project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <rabitqlib/index/query.hpp>
|
||||
namespace zvec::core {
|
||||
|
||||
struct HnswRabitqQueryEntity {
|
||||
std::vector<float> rotated_query;
|
||||
std::vector<float> q_to_centroids;
|
||||
std::unique_ptr<rabitqlib::SplitSingleQuery<float>> query_wrapper;
|
||||
};
|
||||
|
||||
} // namespace zvec::core
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
#include "hnsw_rabitq_builder.h"
|
||||
#include "hnsw_rabitq_searcher.h"
|
||||
#include "hnsw_rabitq_streamer.h"
|
||||
#include "rabitq_converter.h"
|
||||
#include "rabitq_reformer.h"
|
||||
|
||||
namespace zvec::core {
|
||||
|
||||
INDEX_FACTORY_REGISTER_STREAMER(HnswRabitqStreamer);
|
||||
INDEX_FACTORY_REGISTER_REFORMER_ALIAS(RabitqReformer, RabitqReformer);
|
||||
INDEX_FACTORY_REGISTER_SEARCHER(HnswRabitqSearcher);
|
||||
INDEX_FACTORY_REGISTER_CONVERTER_ALIAS(RabitqConverter, RabitqConverter);
|
||||
INDEX_FACTORY_REGISTER_BUILDER(HnswRabitqBuilder);
|
||||
|
||||
} // namespace zvec::core
|
||||
@@ -0,0 +1,516 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_searcher.h"
|
||||
#include <rabitqlib/utils/rotator.hpp>
|
||||
#include "hnsw_rabitq_algorithm.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
#include "hnsw_rabitq_index_provider.h"
|
||||
#include "hnsw_rabitq_params.h"
|
||||
#include "hnsw_rabitq_query_entity.h"
|
||||
#include "hnsw_rabitq_searcher_entity.h"
|
||||
#include "rabitq_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswRabitqSearcher::HnswRabitqSearcher() {}
|
||||
|
||||
HnswRabitqSearcher::~HnswRabitqSearcher() {}
|
||||
|
||||
int HnswRabitqSearcher::init(const ailego::Params &search_params) {
|
||||
params_ = search_params;
|
||||
params_.get(PARAM_HNSW_RABITQ_SEARCHER_EF, &ef_);
|
||||
params_.get(PARAM_HNSW_RABITQ_SEARCHER_MAX_SCAN_RATIO, &max_scan_ratio_);
|
||||
params_.get(PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_ENABLE,
|
||||
&bf_enabled_);
|
||||
params_.get(PARAM_HNSW_RABITQ_SEARCHER_CHECK_CRC_ENABLE, &check_crc_enabled_);
|
||||
params_.get(PARAM_HNSW_RABITQ_SEARCHER_NEIGHBORS_IN_MEMORY_ENABLE,
|
||||
&neighbors_in_memory_enabled_);
|
||||
params_.get(PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB,
|
||||
&bf_negative_probability_);
|
||||
params_.get(PARAM_HNSW_RABITQ_SEARCHER_BRUTE_FORCE_THRESHOLD,
|
||||
&bruteforce_threshold_);
|
||||
params_.get(PARAM_HNSW_RABITQ_SEARCHER_FORCE_PADDING_RESULT_ENABLE,
|
||||
&force_padding_topk_enabled_);
|
||||
|
||||
if (ef_ == 0) {
|
||||
ef_ = HnswRabitqEntity::kDefaultEf;
|
||||
}
|
||||
if (bf_negative_probability_ <= 0.0f || bf_negative_probability_ >= 1.0f) {
|
||||
LOG_ERROR(
|
||||
"[%s] must be in range (0,1)",
|
||||
PARAM_HNSW_RABITQ_SEARCHER_VISIT_BLOOMFILTER_NEGATIVE_PROB.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
entity_.set_neighbors_in_memory(neighbors_in_memory_enabled_);
|
||||
|
||||
ailego::Params reformer_params;
|
||||
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
|
||||
int ret = reformer_.init(reformer_params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
state_ = STATE_INITED;
|
||||
|
||||
LOG_DEBUG(
|
||||
"Init params: ef=%u maxScanRatio=%f bfEnabled=%u checkCrcEnabled=%u "
|
||||
"neighborsInMemoryEnabled=%u bfNagtiveProb=%f bruteForceThreshold=%u "
|
||||
"forcePadding=%u",
|
||||
ef_, max_scan_ratio_, bf_enabled_, check_crc_enabled_,
|
||||
neighbors_in_memory_enabled_, bf_negative_probability_,
|
||||
bruteforce_threshold_, force_padding_topk_enabled_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void HnswRabitqSearcher::print_debug_info() {
|
||||
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
|
||||
Neighbors neighbours = entity_.get_neighbors(0, id);
|
||||
std::cout << "node: " << id << "; ";
|
||||
for (uint32_t i = 0; i < neighbours.size(); ++i) {
|
||||
std::cout << neighbours[i];
|
||||
|
||||
if (i == neighbours.size() - 1) {
|
||||
std::cout << std::endl;
|
||||
} else {
|
||||
std::cout << ", ";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int HnswRabitqSearcher::cleanup() {
|
||||
LOG_INFO("Begin HnswRabitqSearcher:cleanup");
|
||||
|
||||
metric_.reset();
|
||||
meta_.clear();
|
||||
stats_.clear_attributes();
|
||||
stats_.set_loaded_count(0UL);
|
||||
stats_.set_loaded_costtime(0UL);
|
||||
max_scan_ratio_ = HnswRabitqEntity::kDefaultScanRatio;
|
||||
max_scan_num_ = 0U;
|
||||
ef_ = HnswRabitqEntity::kDefaultEf;
|
||||
bf_enabled_ = false;
|
||||
bf_negative_probability_ = HnswRabitqEntity::kDefaultBFNegativeProbability;
|
||||
bruteforce_threshold_ = HnswRabitqEntity::kDefaultBruteForceThreshold;
|
||||
check_crc_enabled_ = false;
|
||||
neighbors_in_memory_enabled_ = false;
|
||||
entity_.cleanup();
|
||||
state_ = STATE_INIT;
|
||||
|
||||
LOG_INFO("End HnswRabitqSearcher:cleanup");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcher::load(IndexStorage::Pointer container,
|
||||
IndexMetric::Pointer metric) {
|
||||
if (state_ != STATE_INITED) {
|
||||
LOG_ERROR("Init the searcher first before load index");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
LOG_INFO("Begin HnswRabitqSearcher:load");
|
||||
|
||||
auto start_time = ailego::Monotime::MilliSeconds();
|
||||
|
||||
int ret = IndexHelper::DeserializeFromStorage(container.get(), &meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to deserialize meta from container");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = reformer_.load(container);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to load reformer from container: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = entity_.load(container, check_crc_enabled_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("HnswRabitqSearcher load index failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
alg_ = HnswRabitqQueryAlgorithm::UPointer(new HnswRabitqQueryAlgorithm(
|
||||
entity_, reformer_.num_clusters(), reformer_.rabitq_metric_type()));
|
||||
|
||||
if (metric) {
|
||||
metric_ = metric;
|
||||
} else {
|
||||
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("CreateMetric failed, name: %s", meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
ret = metric_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
if (metric_->query_metric()) {
|
||||
metric_ = metric_->query_metric();
|
||||
}
|
||||
}
|
||||
|
||||
if (!metric_->is_matched(meta_)) {
|
||||
LOG_ERROR("IndexMetric not match index meta");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
max_scan_num_ = static_cast<uint32_t>(max_scan_ratio_ * entity_.doc_cnt());
|
||||
max_scan_num_ = std::max(4096U, max_scan_num_);
|
||||
|
||||
stats_.set_loaded_count(entity_.doc_cnt());
|
||||
stats_.set_loaded_costtime(ailego::Monotime::MilliSeconds() - start_time);
|
||||
state_ = STATE_LOADED;
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
LOG_INFO("End HnswRabitqSearcher::load");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcher::unload() {
|
||||
LOG_INFO("HnswRabitqSearcher unload index");
|
||||
|
||||
meta_.clear();
|
||||
entity_.cleanup();
|
||||
metric_.reset();
|
||||
max_scan_num_ = 0;
|
||||
stats_.set_loaded_count(0UL);
|
||||
stats_.set_loaded_costtime(0UL);
|
||||
state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcher::update_context(HnswRabitqContext *ctx) const {
|
||||
const HnswRabitqEntity::Pointer entity = entity_.clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone search context entity");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx->set_max_scan_num(max_scan_num_);
|
||||
ctx->set_bruteforce_threshold(bruteforce_threshold_);
|
||||
|
||||
return ctx->update_context(HnswRabitqContext::kSearcherContext, meta_,
|
||||
metric_, entity, magic_);
|
||||
}
|
||||
|
||||
int HnswRabitqSearcher::search_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to HnswRabitqContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (entity_.doc_cnt() <= ctx->get_bruteforce_threshold()) {
|
||||
return search_bf_impl(query, qmeta, count, context);
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer
|
||||
int ret = update_context(ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
int ret = reformer_.transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw searcher transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ret = alg_->search(&entity, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw searcher fast search failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcher::search_bf_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to HnswRabitqContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer
|
||||
int ret = update_context(ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
if (ctx->group_by_search()) {
|
||||
if (!ctx->group_by().is_valid()) {
|
||||
LOG_ERROR("Invalid group-by function");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
|
||||
return ctx->group_by()(entity_.get_key(id));
|
||||
};
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
int ret = reformer_.transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw searcher transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ctx->group_topk_heaps().clear();
|
||||
|
||||
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
|
||||
if (entity_.get_key(id) == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
|
||||
EstimateRecord dist;
|
||||
alg_->get_full_est(id, dist, entity);
|
||||
|
||||
std::string group_id = group_by(id);
|
||||
|
||||
auto &topk_heap = ctx->group_topk_heaps()[group_id];
|
||||
if (topk_heap.empty()) {
|
||||
topk_heap.limit(ctx->group_topk());
|
||||
}
|
||||
topk_heap.emplace_back(id, dist);
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
} else {
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
int ret = reformer_.transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw searcher transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ctx->topk_heap().clear();
|
||||
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
|
||||
if (entity_.get_key(id) == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
|
||||
EstimateRecord dist;
|
||||
alg_->get_full_est(id, dist, entity);
|
||||
ctx->topk_heap().emplace(id, dist);
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcher::search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
if (ailego_unlikely(!query || !context)) {
|
||||
LOG_ERROR("The context is not created by this searcher");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(p_keys.size() != count)) {
|
||||
LOG_ERROR("The size of p_keys is not equal to count");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to HnswRabitqContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer
|
||||
int ret = update_context(ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->resize_results(count);
|
||||
|
||||
if (ctx->group_by_search()) {
|
||||
if (!ctx->group_by().is_valid()) {
|
||||
LOG_ERROR("Invalid group-by function");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
|
||||
return ctx->group_by()(entity_.get_key(id));
|
||||
};
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
int ret = reformer_.transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw searcher transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ctx->group_topk_heaps().clear();
|
||||
|
||||
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
|
||||
uint64_t pk = p_keys[q][idx];
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
|
||||
node_id_t id = entity_.get_id(pk);
|
||||
if (id != kInvalidNodeId) {
|
||||
EstimateRecord dist;
|
||||
alg_->get_full_est(id, dist, entity);
|
||||
std::string group_id = group_by(id);
|
||||
|
||||
auto &topk_heap = ctx->group_topk_heaps()[group_id];
|
||||
if (topk_heap.empty()) {
|
||||
topk_heap.limit(ctx->group_topk());
|
||||
}
|
||||
topk_heap.emplace_back(id, dist);
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
} else {
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
int ret = reformer_.transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw searcher transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ctx->topk_heap().clear();
|
||||
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
|
||||
uint64_t pk = p_keys[q][idx];
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
|
||||
node_id_t id = entity_.get_id(pk);
|
||||
if (id != kInvalidNodeId) {
|
||||
EstimateRecord dist;
|
||||
alg_->get_full_est(id, dist, entity);
|
||||
ctx->topk_heap().emplace(id, dist);
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
IndexSearcher::Context::Pointer HnswRabitqSearcher::create_context() const {
|
||||
if (ailego_unlikely(state_ != STATE_LOADED)) {
|
||||
LOG_ERROR("Load the index first before create context");
|
||||
return Context::Pointer();
|
||||
}
|
||||
const HnswRabitqEntity::Pointer search_ctx_entity = entity_.clone();
|
||||
if (!search_ctx_entity) {
|
||||
LOG_ERROR("Failed to create search context entity");
|
||||
return Context::Pointer();
|
||||
}
|
||||
HnswRabitqContext *ctx = new (std::nothrow)
|
||||
HnswRabitqContext(meta_.dimension(), metric_, search_ctx_entity);
|
||||
if (ailego_unlikely(ctx == nullptr)) {
|
||||
LOG_ERROR("Failed to new HnswRabitqContext");
|
||||
return Context::Pointer();
|
||||
}
|
||||
ctx->set_ef(ef_);
|
||||
ctx->set_max_scan_num(max_scan_num_);
|
||||
uint32_t filter_mode =
|
||||
bf_enabled_ ? VisitFilter::BloomFilter : VisitFilter::ByteMap;
|
||||
ctx->set_filter_mode(filter_mode);
|
||||
ctx->set_filter_negative_probability(bf_negative_probability_);
|
||||
ctx->set_magic(magic_);
|
||||
ctx->set_force_padding_topk(force_padding_topk_enabled_);
|
||||
ctx->set_bruteforce_threshold(bruteforce_threshold_);
|
||||
if (ailego_unlikely(ctx->init(HnswRabitqContext::kSearcherContext)) != 0) {
|
||||
LOG_ERROR("Init HnswRabitqContext failed");
|
||||
delete ctx;
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
return Context::Pointer(ctx);
|
||||
}
|
||||
|
||||
IndexProvider::Pointer HnswRabitqSearcher::create_provider(void) const {
|
||||
LOG_DEBUG("HnswRabitqSearcher create provider");
|
||||
|
||||
auto entity = entity_.clone();
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("Clone HnswRabitqEntity failed");
|
||||
return Provider::Pointer();
|
||||
}
|
||||
return Provider::Pointer(new (std::nothrow) HnswRabitqIndexProvider(
|
||||
meta_, entity, "HnswRabitqSearcher"));
|
||||
}
|
||||
|
||||
const void *HnswRabitqSearcher::get_vector(uint64_t key) const {
|
||||
return entity_.get_vector_by_key(key);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "zvec/core/framework/index_framework.h"
|
||||
#include "hnsw_rabitq_query_algorithm.h"
|
||||
#include "hnsw_rabitq_searcher_entity.h"
|
||||
#include "rabitq_reformer.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class HnswRabitqSearcher : public IndexSearcher {
|
||||
public:
|
||||
using ContextPointer = IndexSearcher::Context::Pointer;
|
||||
|
||||
public:
|
||||
HnswRabitqSearcher(void);
|
||||
~HnswRabitqSearcher(void);
|
||||
|
||||
HnswRabitqSearcher(const HnswRabitqSearcher &) = delete;
|
||||
HnswRabitqSearcher &operator=(const HnswRabitqSearcher &) = delete;
|
||||
|
||||
protected:
|
||||
//! Initialize Searcher
|
||||
virtual int init(const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Searcher
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Load Index from storage
|
||||
virtual int load(IndexStorage::Pointer container,
|
||||
IndexMetric::Pointer metric) override;
|
||||
|
||||
//! Unload index from storage
|
||||
virtual int unload(void) override;
|
||||
|
||||
//! KNN Search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! KNN Search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Linear Search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear Search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Fetch vector by key
|
||||
virtual const void *get_vector(uint64_t key) const override;
|
||||
|
||||
//! Create a searcher context
|
||||
virtual ContextPointer create_context() const override;
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
//! Retrieve params of index
|
||||
virtual const ailego::Params ¶ms(void) const override {
|
||||
return params_;
|
||||
}
|
||||
|
||||
virtual void print_debug_info() override;
|
||||
|
||||
private:
|
||||
//! To share ctx across streamer/searcher, we need to update the context for
|
||||
//! current streamer/searcher
|
||||
int update_context(HnswRabitqContext *ctx) const;
|
||||
|
||||
private:
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_LOADED = 2 };
|
||||
|
||||
HnswRabitqSearcherEntity entity_{};
|
||||
HnswRabitqQueryAlgorithm::UPointer alg_; // impl graph algorithm
|
||||
|
||||
IndexMetric::Pointer metric_{};
|
||||
IndexMeta meta_{};
|
||||
ailego::Params params_{};
|
||||
Stats stats_;
|
||||
uint32_t ef_{HnswRabitqEntity::kDefaultEf};
|
||||
uint32_t max_scan_num_{0U};
|
||||
uint32_t bruteforce_threshold_{HnswRabitqEntity::kDefaultBruteForceThreshold};
|
||||
float max_scan_ratio_{HnswRabitqEntity::kDefaultScanRatio};
|
||||
bool bf_enabled_{false};
|
||||
bool check_crc_enabled_{false};
|
||||
bool neighbors_in_memory_enabled_{false};
|
||||
bool force_padding_topk_enabled_{false};
|
||||
float bf_negative_probability_{
|
||||
HnswRabitqEntity::kDefaultBFNegativeProbability};
|
||||
uint32_t magic_{0U};
|
||||
RabitqReformer reformer_;
|
||||
|
||||
State state_{STATE_INIT};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,515 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_searcher_entity.h"
|
||||
#include <zvec/ailego/hash/crc32c.h>
|
||||
#include "utility/sparse_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswRabitqSearcherEntity::HnswRabitqSearcherEntity() {}
|
||||
|
||||
int HnswRabitqSearcherEntity::cleanup(void) {
|
||||
storage_.reset();
|
||||
vectors_.reset();
|
||||
keys_.reset();
|
||||
neighbors_.reset();
|
||||
neighbors_meta_.reset();
|
||||
neighbors_in_memory_enabled_ = false;
|
||||
loaded_ = false;
|
||||
|
||||
this->HnswRabitqEntity::cleanup();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
key_t HnswRabitqSearcherEntity::get_key(node_id_t id) const {
|
||||
const void *key;
|
||||
if (ailego_unlikely(keys_->read(id * sizeof(key_t), &key, sizeof(key_t)) !=
|
||||
sizeof(key_t))) {
|
||||
LOG_ERROR("Read key from segment failed");
|
||||
return kInvalidKey;
|
||||
}
|
||||
return *(reinterpret_cast<const key_t *>(key));
|
||||
}
|
||||
|
||||
//! Get vector local id by key
|
||||
node_id_t HnswRabitqSearcherEntity::get_id(key_t key) const {
|
||||
if (ailego_unlikely(!mapping_)) {
|
||||
LOG_ERROR("Index missing mapping segment");
|
||||
return kInvalidNodeId;
|
||||
}
|
||||
|
||||
//! Do binary search
|
||||
node_id_t start = 0UL;
|
||||
node_id_t end = doc_cnt();
|
||||
const void *data;
|
||||
node_id_t idx = 0u;
|
||||
while (start < end) {
|
||||
idx = start + (end - start) / 2;
|
||||
if (ailego_unlikely(
|
||||
mapping_->read(idx * sizeof(node_id_t), &data, sizeof(node_id_t)) !=
|
||||
sizeof(node_id_t))) {
|
||||
LOG_ERROR("Read key from segment failed");
|
||||
return kInvalidNodeId;
|
||||
}
|
||||
const key_t *mkey;
|
||||
node_id_t local_id = *reinterpret_cast<const node_id_t *>(data);
|
||||
if (ailego_unlikely(keys_->read(local_id * sizeof(key_t),
|
||||
(const void **)(&mkey),
|
||||
sizeof(key_t)) != sizeof(key_t))) {
|
||||
LOG_ERROR("Read key from segment failed");
|
||||
return kInvalidNodeId;
|
||||
}
|
||||
if (*mkey < key) {
|
||||
start = idx + 1;
|
||||
} else if (*mkey > key) {
|
||||
end = idx;
|
||||
} else {
|
||||
return local_id;
|
||||
}
|
||||
}
|
||||
return kInvalidNodeId;
|
||||
}
|
||||
|
||||
const void *HnswRabitqSearcherEntity::get_vector_by_key(key_t key) const {
|
||||
node_id_t local_id = get_id(key);
|
||||
if (ailego_unlikely(local_id == kInvalidNodeId)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return get_vector(local_id);
|
||||
}
|
||||
|
||||
const void *HnswRabitqSearcherEntity::get_vector(node_id_t id) const {
|
||||
size_t read_size = vector_size();
|
||||
size_t offset = node_size() * id;
|
||||
|
||||
const void *vec;
|
||||
if (ailego_unlikely(vectors_->read(offset, &vec, read_size) != read_size)) {
|
||||
LOG_ERROR("Read vector from segment failed");
|
||||
return nullptr;
|
||||
}
|
||||
return vec;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcherEntity::get_vector(
|
||||
const node_id_t id, IndexStorage::MemoryBlock &block) const {
|
||||
const void *vec = get_vector(id);
|
||||
block.reset((void *)vec);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const void *HnswRabitqSearcherEntity::get_vectors() const {
|
||||
const void *vec;
|
||||
size_t len = node_size() * doc_cnt();
|
||||
if (vectors_->read(0, &vec, len) != len) {
|
||||
LOG_ERROR("Read vectors from segment failed");
|
||||
return nullptr;
|
||||
}
|
||||
return vec;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcherEntity::get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const {
|
||||
ailego_assert_with(count <= segment_datas_.size(), "invalid count");
|
||||
|
||||
size_t read_size = vector_size();
|
||||
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
segment_datas_[i].offset = node_size() * ids[i];
|
||||
segment_datas_[i].length = read_size;
|
||||
|
||||
ailego_assert_with(segment_datas_[i].offset < vectors_->data_size(),
|
||||
"invalid offset");
|
||||
}
|
||||
if (ailego_unlikely(!vectors_->read(&segment_datas_[0], count))) {
|
||||
LOG_ERROR("Read vectors from segment failed");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
vecs[i] = segment_datas_[i].data;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcherEntity::get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
|
||||
const void *vecs[count];
|
||||
get_vector(ids, count, vecs);
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i]));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const Neighbors HnswRabitqSearcherEntity::get_neighbors(level_t level,
|
||||
node_id_t id) const {
|
||||
if (level == 0) {
|
||||
if (neighbors_in_memory_enabled_) {
|
||||
auto hd = reinterpret_cast<const NeighborsHeader *>(
|
||||
fixed_neighbors_.get() + neighbors_size() * id);
|
||||
return {hd->neighbor_cnt, hd->neighbors};
|
||||
}
|
||||
|
||||
const GraphNeighborMeta *m;
|
||||
if (ailego_unlikely(neighbors_meta_->read(id * sizeof(GraphNeighborMeta),
|
||||
(const void **)(&m),
|
||||
sizeof(GraphNeighborMeta)) !=
|
||||
sizeof(GraphNeighborMeta))) {
|
||||
LOG_ERROR("Read neighbors meta from segment failed");
|
||||
return {0, nullptr};
|
||||
}
|
||||
|
||||
const void *data;
|
||||
if (ailego_unlikely(neighbors_->read(m->offset, &data,
|
||||
m->neighbor_cnt * sizeof(node_id_t)) !=
|
||||
m->neighbor_cnt * sizeof(node_id_t))) {
|
||||
LOG_ERROR("Read neighbors from segment failed");
|
||||
return {0, nullptr};
|
||||
}
|
||||
return {static_cast<uint32_t>(m->neighbor_cnt),
|
||||
reinterpret_cast<const node_id_t *>(data)};
|
||||
}
|
||||
|
||||
//! Read level > 0 neighbors
|
||||
const HnswNeighborMeta *m;
|
||||
if (ailego_unlikely(upper_neighbors_meta_->read(id * sizeof(HnswNeighborMeta),
|
||||
(const void **)(&m),
|
||||
sizeof(HnswNeighborMeta)) !=
|
||||
sizeof(HnswNeighborMeta))) {
|
||||
LOG_ERROR("Read neighbors meta from segment failed");
|
||||
return {0, nullptr};
|
||||
}
|
||||
|
||||
ailego_assert_with(level <= m->level, "invalid level");
|
||||
size_t offset = m->offset + (level - 1) * upper_neighbors_size();
|
||||
ailego_assert_with(offset <= upper_neighbors_->data_size(), "invalid offset");
|
||||
const void *data;
|
||||
if (ailego_unlikely(
|
||||
upper_neighbors_->read(offset, &data, upper_neighbors_size()) !=
|
||||
upper_neighbors_size())) {
|
||||
LOG_ERROR("Read neighbors from segment failed");
|
||||
return {0, nullptr};
|
||||
}
|
||||
|
||||
auto hd = reinterpret_cast<const NeighborsHeader *>(data);
|
||||
return {hd->neighbor_cnt, hd->neighbors};
|
||||
}
|
||||
|
||||
int HnswRabitqSearcherEntity::load(const IndexStorage::Pointer &container,
|
||||
bool check_crc) {
|
||||
storage_ = container;
|
||||
|
||||
int ret = load_segments(check_crc);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
loaded_ = true;
|
||||
|
||||
LOG_INFO(
|
||||
"Index info: docCnt=%u entryPoint=%u maxLevel=%d efConstruct=%zu "
|
||||
"l0NeighborCnt=%zu upperNeighborCnt=%zu scalingFactor=%zu "
|
||||
"vectorSize=%zu nodeSize=%zu vectorSegmentSize=%zu keySegmentSize=%zu "
|
||||
"neighborsSegmentSize=%zu neighborsMetaSegmentSize=%zu ",
|
||||
doc_cnt(), entry_point(), cur_max_level(), ef_construction(),
|
||||
l0_neighbor_cnt(), upper_neighbor_cnt(), scaling_factor(), vector_size(),
|
||||
node_size(), vectors_->data_size(), keys_->data_size(),
|
||||
neighbors_ == nullptr ? 0 : neighbors_->data_size(),
|
||||
neighbors_meta_ == nullptr ? 0 : neighbors_meta_->data_size());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcherEntity::load_segments(bool check_crc) {
|
||||
//! load header
|
||||
const void *data = nullptr;
|
||||
HNSWHeader hd;
|
||||
auto graph_hd_segment = storage_->get(kGraphHeaderSegmentId);
|
||||
if (!graph_hd_segment || graph_hd_segment->data_size() < sizeof(hd.graph)) {
|
||||
LOG_ERROR("Miss or invalid segment %s", kGraphHeaderSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (graph_hd_segment->read(0, reinterpret_cast<const void **>(&data),
|
||||
sizeof(hd.graph)) != sizeof(hd.graph)) {
|
||||
LOG_ERROR("Read segment %s failed", kGraphHeaderSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(&hd.graph, data, sizeof(hd.graph));
|
||||
|
||||
auto hnsw_hd_segment = storage_->get(kHnswHeaderSegmentId);
|
||||
if (!hnsw_hd_segment || hnsw_hd_segment->data_size() < sizeof(hd.hnsw)) {
|
||||
LOG_ERROR("Miss or invalid segment %s", kHnswHeaderSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (hnsw_hd_segment->read(0, reinterpret_cast<const void **>(&data),
|
||||
sizeof(hd.hnsw)) != sizeof(hd.hnsw)) {
|
||||
LOG_ERROR("Read segment %s failed", kHnswHeaderSegmentId.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(&hd.hnsw, data, sizeof(hd.hnsw));
|
||||
*mutable_header() = hd;
|
||||
segment_datas_.resize(std::max(l0_neighbor_cnt(), upper_neighbor_cnt()));
|
||||
|
||||
vectors_ = storage_->get(kGraphFeaturesSegmentId);
|
||||
if (!vectors_) {
|
||||
LOG_ERROR("IndexStorage get segment %s failed",
|
||||
kGraphFeaturesSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
keys_ = storage_->get(kGraphKeysSegmentId);
|
||||
if (!keys_) {
|
||||
LOG_ERROR("IndexStorage get segment %s failed",
|
||||
kGraphKeysSegmentId.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
neighbors_ = storage_->get(kGraphNeighborsSegmentId);
|
||||
if (!neighbors_ || (neighbors_->data_size() == 0 && doc_cnt() > 1)) {
|
||||
LOG_ERROR("IndexStorage get segment %s failed or empty",
|
||||
kGraphNeighborsSegmentId.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
neighbors_meta_ = storage_->get(kGraphOffsetsSegmentId);
|
||||
if (!neighbors_meta_ ||
|
||||
neighbors_meta_->data_size() < sizeof(GraphNeighborMeta) * doc_cnt()) {
|
||||
LOG_ERROR("IndexStorage get segment %s failed or invalid size",
|
||||
kGraphOffsetsSegmentId.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
upper_neighbors_ = storage_->get(kHnswNeighborsSegmentId);
|
||||
if (!upper_neighbors_ ||
|
||||
(upper_neighbors_->data_size() == 0 && cur_max_level() > 0)) {
|
||||
LOG_ERROR("IndexStorage get segment %s failed or empty",
|
||||
kHnswNeighborsSegmentId.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
upper_neighbors_meta_ = storage_->get(kHnswOffsetsSegmentId);
|
||||
if (!upper_neighbors_meta_ || upper_neighbors_meta_->data_size() <
|
||||
sizeof(HnswNeighborMeta) * doc_cnt()) {
|
||||
LOG_ERROR("IndexStorage get segment %s failed or invalid size",
|
||||
kHnswOffsetsSegmentId.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
mapping_ = storage_->get(kGraphMappingSegmentId);
|
||||
if (!mapping_ || mapping_->data_size() < sizeof(node_id_t) * doc_cnt()) {
|
||||
LOG_ERROR("IndexStorage get segment %s failed or invalid size",
|
||||
kGraphMappingSegmentId.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (check_crc) {
|
||||
std::vector<SegmentPointer> segments;
|
||||
segments.emplace_back(graph_hd_segment);
|
||||
segments.emplace_back(hnsw_hd_segment);
|
||||
segments.emplace_back(vectors_);
|
||||
segments.emplace_back(keys_);
|
||||
|
||||
segments.emplace_back(neighbors_);
|
||||
segments.emplace_back(neighbors_meta_);
|
||||
segments.emplace_back(upper_neighbors_);
|
||||
segments.emplace_back(upper_neighbors_meta_);
|
||||
|
||||
if (!do_crc_check(segments)) {
|
||||
LOG_ERROR("Check index crc failed, the index may broken");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
|
||||
if (neighbors_in_memory_enabled_) {
|
||||
int ret = load_and_flat_neighbors();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcherEntity::load_and_flat_neighbors() {
|
||||
fixed_neighbors_.reset(
|
||||
new (std::nothrow) char[neighbors_size() * doc_cnt()]{},
|
||||
std::default_delete<char[]>());
|
||||
if (!fixed_neighbors_) {
|
||||
LOG_ERROR("Malloc memory failed");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
|
||||
//! Get a new segemnt to release the buffer after loading neighbors
|
||||
auto neighbors_meta = storage_->get(kGraphOffsetsSegmentId);
|
||||
if (!neighbors_meta) {
|
||||
LOG_ERROR("IndexStorage get segment graph.offsets failed");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
const GraphNeighborMeta *neighbors_index = nullptr;
|
||||
if (neighbors_meta->read(0, reinterpret_cast<const void **>(&neighbors_index),
|
||||
neighbors_meta->data_size()) !=
|
||||
neighbors_meta->data_size()) {
|
||||
LOG_ERROR("Read segment %s data failed", kGraphOffsetsSegmentId.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
const char *neighbor_data;
|
||||
for (node_id_t id = 0; id < doc_cnt(); ++id) {
|
||||
size_t rd_size = neighbors_index[id].neighbor_cnt * sizeof(node_id_t);
|
||||
if (ailego_unlikely(
|
||||
neighbors_->read(neighbors_index[id].offset,
|
||||
reinterpret_cast<const void **>(&neighbor_data),
|
||||
rd_size) != rd_size)) {
|
||||
LOG_ERROR("Read neighbors from segment failed");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
// copy level 0 neighbors to fixed size neighbors memory
|
||||
char *dst = fixed_neighbors_.get() + neighbors_size() * id;
|
||||
*reinterpret_cast<uint32_t *>(dst) = neighbors_index[id].neighbor_cnt;
|
||||
memcpy(dst + sizeof(uint32_t), neighbor_data, rd_size);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqSearcherEntity::get_fixed_neighbors(
|
||||
std::vector<uint32_t> *fixed_neighbors) const {
|
||||
//! Get a new segemnt to release the buffer after loading neighbors
|
||||
auto neighbors_meta = storage_->get(kGraphOffsetsSegmentId);
|
||||
if (!neighbors_meta) {
|
||||
LOG_ERROR("IndexStorage get segment graph.offsets failed");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
const GraphNeighborMeta *neighbors_index = nullptr;
|
||||
size_t meta_size = neighbors_meta->data_size();
|
||||
if (neighbors_meta->read(0, reinterpret_cast<const void **>(&neighbors_index),
|
||||
meta_size) != meta_size) {
|
||||
LOG_ERROR("Read segment %s data failed", kGraphOffsetsSegmentId.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
size_t fixed_neighbor_cnt = l0_neighbor_cnt();
|
||||
fixed_neighbors->resize((fixed_neighbor_cnt + 1) * doc_cnt(), kInvalidNodeId);
|
||||
|
||||
size_t neighbors_cnt_offset = fixed_neighbor_cnt * doc_cnt();
|
||||
size_t total_neighbor_cnt = 0;
|
||||
for (node_id_t id = 0; id < doc_cnt(); ++id) {
|
||||
size_t cur_neighbor_cnt = neighbors_index[id].neighbor_cnt;
|
||||
if (cur_neighbor_cnt == 0) {
|
||||
(*fixed_neighbors)[neighbors_cnt_offset + id] = 0;
|
||||
continue;
|
||||
}
|
||||
size_t rd_size = cur_neighbor_cnt * sizeof(node_id_t);
|
||||
const uint32_t *neighbors;
|
||||
if (neighbors_->read(neighbors_index[id].offset,
|
||||
reinterpret_cast<const void **>(&neighbors),
|
||||
rd_size) != rd_size) {
|
||||
LOG_ERROR("Read neighbors from segment failed");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
// copy level 0 neighbors to fixed size neighbors memory
|
||||
auto it = fixed_neighbors->begin() + id * fixed_neighbor_cnt;
|
||||
std::copy(neighbors, neighbors + cur_neighbor_cnt, it);
|
||||
|
||||
(*fixed_neighbors)[neighbors_cnt_offset + id] = cur_neighbor_cnt;
|
||||
total_neighbor_cnt += cur_neighbor_cnt;
|
||||
}
|
||||
LOG_INFO("total neighbor cnt: %zu, average neighbor cnt: %zu",
|
||||
total_neighbor_cnt, total_neighbor_cnt / doc_cnt());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool HnswRabitqSearcherEntity::do_crc_check(
|
||||
std::vector<SegmentPointer> &segments) const {
|
||||
constexpr size_t blk_size = 4096;
|
||||
const void *data;
|
||||
for (auto &segment : segments) {
|
||||
size_t offset = 0;
|
||||
size_t rd_size;
|
||||
uint32_t crc = 0;
|
||||
while (offset < segment->data_size()) {
|
||||
size_t size = std::min(blk_size, segment->data_size() - offset);
|
||||
if ((rd_size = segment->read(offset, &data, size)) <= 0) {
|
||||
break;
|
||||
}
|
||||
offset += rd_size;
|
||||
crc = ailego::Crc32c::Hash(data, rd_size, crc);
|
||||
}
|
||||
if (crc != segment->data_crc()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const HnswRabitqEntity::Pointer HnswRabitqSearcherEntity::clone() const {
|
||||
auto vectors = vectors_->clone();
|
||||
if (ailego_unlikely(!vectors)) {
|
||||
LOG_ERROR("clone segment %s failed", kGraphFeaturesSegmentId.c_str());
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
auto keys = keys_->clone();
|
||||
if (ailego_unlikely(!keys)) {
|
||||
LOG_ERROR("clone segment %s failed", kGraphKeysSegmentId.c_str());
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
|
||||
auto mapping = mapping_->clone();
|
||||
if (ailego_unlikely(!mapping)) {
|
||||
LOG_ERROR("clone segment %s failed", kGraphMappingSegmentId.c_str());
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
|
||||
auto neighbors = neighbors_->clone();
|
||||
if (ailego_unlikely(!neighbors)) {
|
||||
LOG_ERROR("clone segment %s failed", kGraphNeighborsSegmentId.c_str());
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
auto upper_neighbors = upper_neighbors_->clone();
|
||||
if (ailego_unlikely(!neighbors)) {
|
||||
LOG_ERROR("clone segment %s failed", kHnswNeighborsSegmentId.c_str());
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
auto neighbors_meta = neighbors_meta_->clone();
|
||||
if (ailego_unlikely(!neighbors_meta)) {
|
||||
LOG_ERROR("clone segment %s failed", kGraphOffsetsSegmentId.c_str());
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
auto upper_neighbors_meta = upper_neighbors_meta_->clone();
|
||||
if (ailego_unlikely(!upper_neighbors_meta)) {
|
||||
LOG_ERROR("clone segment %s failed", kHnswOffsetsSegmentId.c_str());
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
|
||||
SegmentGroupParam neighbor_group{neighbors, neighbors_meta, upper_neighbors,
|
||||
upper_neighbors_meta};
|
||||
|
||||
HnswRabitqSearcherEntity *entity = new (std::nothrow)
|
||||
HnswRabitqSearcherEntity(header(), vectors, keys, mapping, neighbor_group,
|
||||
fixed_neighbors_, neighbors_in_memory_enabled_);
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("HnswRabitqSearcherEntity new failed");
|
||||
}
|
||||
|
||||
return HnswRabitqEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,158 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "hnsw_rabitq_builder_entity.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class HnswRabitqSearcherEntity : public HnswRabitqEntity {
|
||||
public:
|
||||
using Pointer = std::shared_ptr<HnswRabitqSearcherEntity>;
|
||||
using SegmentPointer = IndexStorage::Segment::Pointer;
|
||||
|
||||
public:
|
||||
struct SegmentGroupParam {
|
||||
SegmentGroupParam(SegmentPointer neighbors_in,
|
||||
SegmentPointer neighbors_meta_in,
|
||||
SegmentPointer upper_neighbors_in,
|
||||
SegmentPointer upper_neighbors_meta_in)
|
||||
: neighbors{neighbors_in},
|
||||
neighbors_meta{neighbors_meta_in},
|
||||
upper_neighbors{upper_neighbors_in},
|
||||
upper_neighbors_meta{upper_neighbors_meta_in} {}
|
||||
|
||||
SegmentPointer neighbors{nullptr};
|
||||
SegmentPointer neighbors_meta{nullptr};
|
||||
SegmentPointer upper_neighbors{nullptr};
|
||||
SegmentPointer upper_neighbors_meta{nullptr};
|
||||
};
|
||||
|
||||
//! Constructor
|
||||
HnswRabitqSearcherEntity();
|
||||
|
||||
//! Make a copy of searcher entity, to support thread-safe operation.
|
||||
//! The segment in container cannot be read concurrenly
|
||||
virtual const HnswRabitqEntity::Pointer clone() const override;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
|
||||
//! Get vector local id by key
|
||||
node_id_t get_id(key_t key) const;
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector_by_key(key_t key) const override;
|
||||
|
||||
//! Get vector feature data by id
|
||||
virtual const void *get_vector(node_id_t id) const override;
|
||||
|
||||
//! Get vector feature data by id
|
||||
virtual int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
|
||||
virtual int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
virtual int get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
|
||||
|
||||
//! Get all vectors
|
||||
const void *get_vectors() const;
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
virtual const Neighbors get_neighbors(level_t level,
|
||||
node_id_t id) const override;
|
||||
|
||||
virtual int load(const IndexStorage::Pointer &container,
|
||||
bool check_crc) override;
|
||||
|
||||
int load_segments(bool check_crc);
|
||||
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
public:
|
||||
bool is_loaded() const {
|
||||
return loaded_;
|
||||
}
|
||||
|
||||
void set_neighbors_in_memory(bool enabled) {
|
||||
neighbors_in_memory_enabled_ = enabled;
|
||||
}
|
||||
|
||||
//! get fixed length neighbors data
|
||||
int get_fixed_neighbors(std::vector<uint32_t> *fixed_neighbors) const;
|
||||
|
||||
private:
|
||||
//! Constructor
|
||||
HnswRabitqSearcherEntity(const HNSWHeader &hd, const SegmentPointer &vectors,
|
||||
const SegmentPointer &keys,
|
||||
const SegmentPointer &mapping,
|
||||
const SegmentGroupParam &neighbor_group,
|
||||
const std::shared_ptr<char> &fixed_neighbors,
|
||||
bool neighbors_in_memory_enabled)
|
||||
: HnswRabitqEntity(hd),
|
||||
vectors_(vectors),
|
||||
keys_(keys),
|
||||
mapping_(mapping),
|
||||
neighbors_(neighbor_group.neighbors),
|
||||
neighbors_meta_(neighbor_group.neighbors_meta),
|
||||
upper_neighbors_(neighbor_group.upper_neighbors),
|
||||
upper_neighbors_meta_(neighbor_group.upper_neighbors_meta),
|
||||
neighbors_in_memory_enabled_(neighbors_in_memory_enabled) {
|
||||
segment_datas_.resize(std::max(l0_neighbor_cnt(), upper_neighbor_cnt()),
|
||||
IndexStorage::SegmentData(0U, 0U));
|
||||
fixed_neighbors_ = fixed_neighbors;
|
||||
}
|
||||
|
||||
bool do_crc_check(std::vector<SegmentPointer> &segments) const;
|
||||
|
||||
inline size_t neighbors_size() const {
|
||||
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
|
||||
}
|
||||
|
||||
inline size_t upper_neighbors_size() const {
|
||||
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
|
||||
}
|
||||
|
||||
//! If neighbors_in_memory_enabled, load the level0 neighbors to memory
|
||||
int load_and_flat_neighbors(void);
|
||||
|
||||
public:
|
||||
HnswRabitqSearcherEntity(const HnswRabitqSearcherEntity &) = delete;
|
||||
HnswRabitqSearcherEntity &operator=(const HnswRabitqSearcherEntity &) =
|
||||
delete;
|
||||
|
||||
private:
|
||||
IndexStorage::Pointer storage_{};
|
||||
|
||||
SegmentPointer vectors_{};
|
||||
SegmentPointer keys_{};
|
||||
SegmentPointer mapping_{};
|
||||
|
||||
SegmentPointer neighbors_{};
|
||||
SegmentPointer neighbors_meta_{};
|
||||
SegmentPointer upper_neighbors_{};
|
||||
SegmentPointer upper_neighbors_meta_{};
|
||||
|
||||
mutable std::vector<IndexStorage::SegmentData> segment_datas_{};
|
||||
std::shared_ptr<char> fixed_neighbors_{}; // level 0 fixed size neighbors
|
||||
bool neighbors_in_memory_enabled_{false};
|
||||
bool loaded_{false};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,989 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_rabitq_streamer.h"
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include <ailego/pattern/defer.h>
|
||||
#include <ailego/utility/memory_helper.h>
|
||||
#include <arrow/compute/ordering.h>
|
||||
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
|
||||
#include "zvec/ailego/container/params.h"
|
||||
#include "zvec/ailego/logger/logger.h"
|
||||
#include "hnsw_rabitq_algorithm.h"
|
||||
#include "hnsw_rabitq_context.h"
|
||||
#include "hnsw_rabitq_dist_calculator.h"
|
||||
#include "hnsw_rabitq_index_provider.h"
|
||||
#include "hnsw_rabitq_query_entity.h"
|
||||
#include "rabitq_params.h"
|
||||
#include "rabitq_utils.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
HnswRabitqStreamer::HnswRabitqStreamer() : entity_(stats_) {}
|
||||
|
||||
HnswRabitqStreamer::HnswRabitqStreamer(IndexProvider::Pointer provider,
|
||||
RabitqReformer::Pointer reformer)
|
||||
: entity_(stats_),
|
||||
reformer_(std::move(reformer)),
|
||||
provider_(std::move(provider)) {}
|
||||
|
||||
HnswRabitqStreamer::~HnswRabitqStreamer() {
|
||||
if (state_ == STATE_INITED) {
|
||||
this->cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::init(const IndexMeta &imeta,
|
||||
const ailego::Params ¶ms) {
|
||||
meta_ = imeta;
|
||||
meta_.set_streamer("HnswRabitqStreamer", HnswRabitqEntity::kRevision, params);
|
||||
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_INDEX_SIZE, &max_index_size_);
|
||||
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT,
|
||||
&upper_max_neighbor_cnt_);
|
||||
float multiplier = HnswRabitqEntity::kDefaultL0MaxNeighborCntMultiplier;
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER,
|
||||
&multiplier);
|
||||
l0_max_neighbor_cnt_ = multiplier * upper_max_neighbor_cnt_;
|
||||
|
||||
multiplier = HnswRabitqEntity::kDefaultNeighborPruneMultiplier;
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_NEIGHBOR_PRUNE_MULTIPLIER, &multiplier);
|
||||
size_t prune_cnt = multiplier * upper_max_neighbor_cnt_;
|
||||
scaling_factor_ = upper_max_neighbor_cnt_;
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_SCALING_FACTOR, &scaling_factor_);
|
||||
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT, &docs_hard_limit_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_EF, &ef_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_EFCONSTRUCTION, &ef_construction_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_ENABLE, &bf_enabled_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB,
|
||||
&bf_negative_prob_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_BRUTE_FORCE_THRESHOLD,
|
||||
&bruteforce_threshold_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_CHECK_CRC_ENABLE, &check_crc_enabled_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_CHUNK_SIZE, &chunk_size_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_FILTER_SAME_KEY, &filter_same_key_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_GET_VECTOR_ENABLE,
|
||||
&get_vector_enabled_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_MIN_NEIGHBOR_COUNT, &min_neighbor_cnt_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_FORCE_PADDING_RESULT_ENABLE,
|
||||
&force_padding_topk_enabled_);
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_USE_ID_MAP, &use_id_map_);
|
||||
entity_.set_use_key_info_map(use_id_map_);
|
||||
|
||||
params.get(PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT, &docs_soft_limit_);
|
||||
if (docs_soft_limit_ > 0 && docs_soft_limit_ > docs_hard_limit_) {
|
||||
LOG_ERROR("[%s] must be >= [%s]",
|
||||
PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT.c_str(),
|
||||
PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
} else if (docs_soft_limit_ == 0UL) {
|
||||
docs_soft_limit_ =
|
||||
docs_hard_limit_ * HnswRabitqEntity::kDefaultDocsSoftLimitRatio;
|
||||
}
|
||||
|
||||
if (ef_ == 0U) {
|
||||
ef_ = HnswRabitqEntity::kDefaultEf;
|
||||
}
|
||||
if (ef_construction_ == 0U) {
|
||||
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
|
||||
}
|
||||
if (upper_max_neighbor_cnt_ == 0U) {
|
||||
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
|
||||
}
|
||||
if (upper_max_neighbor_cnt_ > HnswRabitqEntity::kMaxNeighborCnt) {
|
||||
LOG_ERROR("[%s] must be in range (0,%d)",
|
||||
PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT.c_str(),
|
||||
HnswRabitqEntity::kMaxNeighborCnt);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (l0_max_neighbor_cnt_ == 0U) {
|
||||
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultL0MaxNeighborCnt;
|
||||
}
|
||||
if (l0_max_neighbor_cnt_ > HnswRabitqEntity::kMaxNeighborCnt) {
|
||||
LOG_ERROR("MaxL0NeighborCnt must be in range (0,%d)",
|
||||
HnswRabitqEntity::kMaxNeighborCnt);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (min_neighbor_cnt_ > upper_max_neighbor_cnt_) {
|
||||
LOG_ERROR("[%s]-[%zu] must be <= [%s]-[%zu]",
|
||||
PARAM_HNSW_RABITQ_STREAMER_MIN_NEIGHBOR_COUNT.c_str(),
|
||||
static_cast<size_t>(min_neighbor_cnt_),
|
||||
PARAM_HNSW_RABITQ_STREAMER_MAX_NEIGHBOR_COUNT.c_str(),
|
||||
static_cast<size_t>(upper_max_neighbor_cnt_));
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (bf_negative_prob_ <= 0.0f || bf_negative_prob_ >= 1.0f) {
|
||||
LOG_ERROR(
|
||||
"[%s] must be in range (0,1)",
|
||||
PARAM_HNSW_RABITQ_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (scaling_factor_ == 0U) {
|
||||
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
|
||||
}
|
||||
if (scaling_factor_ < 5 || scaling_factor_ > 1000) {
|
||||
LOG_ERROR("[%s] must be in range [5,1000]",
|
||||
PARAM_HNSW_RABITQ_STREAMER_SCALING_FACTOR.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (max_scan_ratio_ <= 0.0f || max_scan_ratio_ > 1.0f) {
|
||||
LOG_ERROR("[%s] must be in range (0.0f,1.0f]",
|
||||
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_RATIO.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (max_scan_limit_ < min_scan_limit_) {
|
||||
LOG_ERROR("[%s] must be >= [%s]",
|
||||
PARAM_HNSW_RABITQ_STREAMER_MAX_SCAN_LIMIT.c_str(),
|
||||
PARAM_HNSW_RABITQ_STREAMER_MIN_SCAN_LIMIT.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (prune_cnt == 0UL) {
|
||||
prune_cnt = upper_max_neighbor_cnt_;
|
||||
}
|
||||
if (chunk_size_ == 0UL) {
|
||||
chunk_size_ = HnswRabitqEntity::kDefaultChunkSize;
|
||||
}
|
||||
if (chunk_size_ > HnswRabitqEntity::kMaxChunkSize) {
|
||||
LOG_ERROR("[%s] must be < %zu",
|
||||
PARAM_HNSW_RABITQ_STREAMER_CHUNK_SIZE.c_str(),
|
||||
HnswRabitqEntity::kMaxChunkSize);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
uint32_t total_bits = 0;
|
||||
params.get(PARAM_RABITQ_TOTAL_BITS, &total_bits);
|
||||
if (total_bits == 0) {
|
||||
total_bits = kDefaultRabitqTotalBits;
|
||||
}
|
||||
if (total_bits < 1 || total_bits > 9) {
|
||||
LOG_ERROR("Invalid total_bits: %zu, must be in [1, 9]", (size_t)total_bits);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
uint8_t ex_bits = total_bits - 1;
|
||||
entity_.set_ex_bits(ex_bits);
|
||||
|
||||
uint32_t dimension = 0;
|
||||
params.get(PARAM_HNSW_RABITQ_GENERAL_DIMENSION, &dimension);
|
||||
if (dimension == 0) {
|
||||
LOG_ERROR("%s not set", PARAM_HNSW_RABITQ_GENERAL_DIMENSION.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (dimension < kMinRabitqDimSize || dimension > kMaxRabitqDimSize) {
|
||||
LOG_ERROR("Invalid dimension: %u, must be in [%d, %d]", dimension,
|
||||
kMinRabitqDimSize, kMaxRabitqDimSize);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
entity_.update_rabitq_params_and_vector_size(dimension);
|
||||
|
||||
entity_.set_ef_construction(ef_construction_);
|
||||
entity_.set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
|
||||
entity_.set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
|
||||
entity_.set_scaling_factor(scaling_factor_);
|
||||
entity_.set_prune_cnt(prune_cnt);
|
||||
|
||||
entity_.set_chunk_size(chunk_size_);
|
||||
entity_.set_filter_same_key(filter_same_key_);
|
||||
entity_.set_get_vector(get_vector_enabled_);
|
||||
entity_.set_min_neighbor_cnt(min_neighbor_cnt_);
|
||||
|
||||
int ret = entity_.init(docs_hard_limit_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Hnsw entity init failed for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_DEBUG(
|
||||
"Init params: maxIndexSize=%zu docsHardLimit=%zu docsSoftLimit=%zu "
|
||||
"efConstruction=%u ef=%u upperMaxNeighborCnt=%u l0MaxNeighborCnt=%u "
|
||||
"scalingFactor=%u maxScanRatio=%.3f minScanLimit=%zu maxScanLimit=%zu "
|
||||
"bfEnabled=%d bruteFoceThreshold=%zu bfNegativeProbability=%.5f "
|
||||
"checkCrcEnabled=%d pruneSize=%zu vectorSize=%u chunkSize=%zu "
|
||||
"filterSameKey=%u getVectorEnabled=%u minNeighborCount=%u "
|
||||
"forcePadding=%u ",
|
||||
max_index_size_, docs_hard_limit_, docs_soft_limit_, ef_construction_,
|
||||
ef_, upper_max_neighbor_cnt_, l0_max_neighbor_cnt_, scaling_factor_,
|
||||
max_scan_ratio_, min_scan_limit_, max_scan_limit_, bf_enabled_,
|
||||
bruteforce_threshold_, bf_negative_prob_, check_crc_enabled_, prune_cnt,
|
||||
meta_.element_size(), chunk_size_, filter_same_key_, get_vector_enabled_,
|
||||
min_neighbor_cnt_, force_padding_topk_enabled_);
|
||||
|
||||
alg_ = HnswRabitqAlgorithm::UPointer(new HnswRabitqAlgorithm(entity_));
|
||||
|
||||
ret = alg_->init();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::cleanup(void) {
|
||||
if (state_ == STATE_OPENED) {
|
||||
this->close();
|
||||
}
|
||||
|
||||
LOG_INFO("HnswRabitqStreamer cleanup");
|
||||
|
||||
meta_.clear();
|
||||
metric_.reset();
|
||||
stats_.clear();
|
||||
entity_.cleanup();
|
||||
|
||||
if (alg_) {
|
||||
alg_->cleanup();
|
||||
}
|
||||
|
||||
max_index_size_ = 0UL;
|
||||
docs_hard_limit_ = HnswRabitqEntity::kDefaultDocsHardLimit;
|
||||
docs_soft_limit_ = 0UL;
|
||||
upper_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultUpperMaxNeighborCnt;
|
||||
l0_max_neighbor_cnt_ = HnswRabitqEntity::kDefaultL0MaxNeighborCnt;
|
||||
ef_ = HnswRabitqEntity::kDefaultEf;
|
||||
ef_construction_ = HnswRabitqEntity::kDefaultEfConstruction;
|
||||
bf_enabled_ = false;
|
||||
scaling_factor_ = HnswRabitqEntity::kDefaultScalingFactor;
|
||||
bruteforce_threshold_ = HnswRabitqEntity::kDefaultBruteForceThreshold;
|
||||
max_scan_limit_ = HnswRabitqEntity::kDefaultMaxScanLimit;
|
||||
min_scan_limit_ = HnswRabitqEntity::kDefaultMinScanLimit;
|
||||
chunk_size_ = HnswRabitqEntity::kDefaultChunkSize;
|
||||
bf_negative_prob_ = HnswRabitqEntity::kDefaultBFNegativeProbability;
|
||||
max_scan_ratio_ = HnswRabitqEntity::kDefaultScanRatio;
|
||||
state_ = STATE_INIT;
|
||||
check_crc_enabled_ = false;
|
||||
filter_same_key_ = false;
|
||||
get_vector_enabled_ = false;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::open(IndexStorage::Pointer stg) {
|
||||
LOG_INFO("HnswRabitqStreamer open");
|
||||
|
||||
if (ailego_unlikely(state_ != STATE_INITED)) {
|
||||
LOG_ERROR("Open storage failed, init streamer first!");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
// try to load reformer
|
||||
if (reformer_ == nullptr) {
|
||||
reformer_ = std::make_shared<RabitqReformer>();
|
||||
ailego::Params reformer_params;
|
||||
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
|
||||
int ret = reformer_->init(reformer_params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = reformer_->load(stg);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to load reformer, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
if (!stg->has(RABITQ_CONVERTER_SEG_ID)) {
|
||||
int ret = reformer_->dump(stg);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to dump reformer, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
LOG_INFO("Dump reformer success.");
|
||||
}
|
||||
}
|
||||
|
||||
int ret = entity_.open(std::move(stg), max_index_size_, check_crc_enabled_);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
IndexMeta index_meta;
|
||||
ret = entity_.get_index_meta(&index_meta);
|
||||
if (ret == IndexError_NoExist) {
|
||||
// Set IndexMeta for the new index
|
||||
ret = entity_.set_index_meta(meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to set index meta for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
} else if (ret != 0) {
|
||||
LOG_ERROR("Failed to get index meta for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
} else {
|
||||
if (index_meta.dimension() != meta_.dimension() ||
|
||||
index_meta.element_size() != meta_.element_size() ||
|
||||
index_meta.metric_name() != meta_.metric_name() ||
|
||||
index_meta.data_type() != meta_.data_type()) {
|
||||
LOG_ERROR("IndexMeta mismatch from the previous in index");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
// The IndexMetric Params may be updated like MipsSquaredEuclidean
|
||||
auto metric_params = index_meta.metric_params();
|
||||
metric_params.merge(meta_.metric_params());
|
||||
meta_.set_metric(index_meta.metric_name(), 0, metric_params);
|
||||
}
|
||||
|
||||
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("Failed to create metric %s", meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
ret = metric_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to init metric, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (!metric_->distance()) {
|
||||
LOG_ERROR("Invalid metric distance");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!metric_->batch_distance()) {
|
||||
LOG_ERROR("Invalid metric batch distance");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
add_distance_ = metric_->distance();
|
||||
add_batch_distance_ = metric_->batch_distance();
|
||||
|
||||
search_distance_ = add_distance_;
|
||||
search_batch_distance_ = add_batch_distance_;
|
||||
|
||||
if (metric_->query_metric() && metric_->query_metric()->distance() &&
|
||||
metric_->query_metric()->batch_distance()) {
|
||||
search_distance_ = metric_->query_metric()->distance();
|
||||
search_batch_distance_ = metric_->query_metric()->batch_distance();
|
||||
}
|
||||
|
||||
state_ = STATE_OPENED;
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
query_alg_ = HnswRabitqQueryAlgorithm::UPointer(new HnswRabitqQueryAlgorithm(
|
||||
entity_, reformer_->num_clusters(), reformer_->rabitq_metric_type()));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::close(void) {
|
||||
LOG_INFO("HnswRabitqStreamer close");
|
||||
|
||||
stats_.clear();
|
||||
meta_.set_metric(metric_->name(), 0, metric_->params());
|
||||
entity_.set_index_meta(meta_);
|
||||
int ret = entity_.close();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::flush(uint64_t checkpoint) {
|
||||
LOG_INFO("HnswRabitqStreamer flush checkpoint=%zu", (size_t)checkpoint);
|
||||
|
||||
meta_.set_metric(metric_->name(), 0, metric_->params());
|
||||
entity_.set_index_meta(meta_);
|
||||
return entity_.flush(checkpoint);
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::dump(const IndexDumper::Pointer &dumper) {
|
||||
LOG_INFO("HnswRabitqStreamer dump");
|
||||
|
||||
shared_mutex_.lock();
|
||||
AILEGO_DEFER([&]() { shared_mutex_.unlock(); });
|
||||
|
||||
meta_.set_searcher("HnswRabitqSearcher", HnswRabitqEntity::kRevision,
|
||||
ailego::Params());
|
||||
|
||||
int ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to serialize meta into dumper.");
|
||||
return ret;
|
||||
}
|
||||
ret = reformer_->dump(dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to dump reformer into dumper.");
|
||||
return ret;
|
||||
}
|
||||
return entity_.dump(dumper);
|
||||
}
|
||||
|
||||
IndexStreamer::Context::Pointer HnswRabitqStreamer::create_context(void) const {
|
||||
if (ailego_unlikely(state_ != STATE_OPENED)) {
|
||||
LOG_ERROR("Create context failed, open storage first!");
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
HnswRabitqEntity::Pointer entity = entity_.clone();
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("CreateContext clone init failed");
|
||||
return Context::Pointer();
|
||||
}
|
||||
HnswRabitqContext *ctx =
|
||||
new (std::nothrow) HnswRabitqContext(meta_.dimension(), metric_, entity);
|
||||
if (ailego_unlikely(ctx == nullptr)) {
|
||||
LOG_ERROR("Failed to new HnswRabitqContext");
|
||||
return Context::Pointer();
|
||||
}
|
||||
ctx->set_ef(ef_);
|
||||
ctx->set_max_scan_limit(max_scan_limit_);
|
||||
ctx->set_min_scan_limit(min_scan_limit_);
|
||||
ctx->set_max_scan_ratio(max_scan_ratio_);
|
||||
ctx->set_filter_mode(bf_enabled_ ? VisitFilter::BloomFilter
|
||||
: VisitFilter::ByteMap);
|
||||
ctx->set_filter_negative_probability(bf_negative_prob_);
|
||||
ctx->set_magic(magic_);
|
||||
ctx->set_force_padding_topk(force_padding_topk_enabled_);
|
||||
ctx->set_bruteforce_threshold(bruteforce_threshold_);
|
||||
|
||||
if (ailego_unlikely(ctx->init(HnswRabitqContext::kStreamerContext)) != 0) {
|
||||
LOG_ERROR("Init HnswRabitqContext failed");
|
||||
delete ctx;
|
||||
return Context::Pointer();
|
||||
}
|
||||
uint32_t estimate_doc_count = 0;
|
||||
if (meta_.streamer_params().get(PARAM_HNSW_RABITQ_STREAMER_ESTIMATE_DOC_COUNT,
|
||||
&estimate_doc_count)) {
|
||||
LOG_DEBUG("HnswRabitqStreamer doc_count[%zu] estimate[%zu]",
|
||||
(size_t)entity_.doc_cnt(), (size_t)estimate_doc_count);
|
||||
}
|
||||
ctx->check_need_adjuct_ctx(std::max(entity_.doc_cnt(), estimate_doc_count));
|
||||
|
||||
return Context::Pointer(ctx);
|
||||
}
|
||||
|
||||
IndexProvider::Pointer HnswRabitqStreamer::create_provider(void) const {
|
||||
LOG_DEBUG("HnswRabitqStreamer create provider");
|
||||
|
||||
auto entity = entity_.clone();
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("Clone HnswRabitqEntity failed");
|
||||
return nullptr;
|
||||
}
|
||||
return Provider::Pointer(
|
||||
new HnswRabitqIndexProvider(meta_, entity, "HnswRabitqStreamer"));
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::update_context(HnswRabitqContext *ctx) const {
|
||||
const HnswRabitqEntity::Pointer entity = entity_.clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone search context entity");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx->set_max_scan_limit(max_scan_limit_);
|
||||
ctx->set_min_scan_limit(min_scan_limit_);
|
||||
ctx->set_max_scan_ratio(max_scan_ratio_);
|
||||
ctx->set_bruteforce_threshold(bruteforce_threshold_);
|
||||
return ctx->update_context(HnswRabitqContext::kStreamerContext, meta_,
|
||||
metric_, entity, magic_);
|
||||
}
|
||||
|
||||
//! Add a vector with id into index
|
||||
int HnswRabitqStreamer::add_with_id_impl(
|
||||
uint32_t id, const void *query, const IndexQueryMeta &qmeta,
|
||||
IndexStreamer::Context::Pointer &context) {
|
||||
if (!provider_) {
|
||||
LOG_ERROR("Provider is nullptr, cannot add vector");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to HnswRabitqContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
|
||||
if (entity_.doc_cnt() >= docs_hard_limit_) {
|
||||
LOG_ERROR("Current docs %zu exceed [%s]",
|
||||
static_cast<size_t>(entity_.doc_cnt()),
|
||||
PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT.c_str());
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_IndexFull;
|
||||
} else {
|
||||
LOG_WARN("Current docs %zu exceed [%s]",
|
||||
static_cast<size_t>(entity_.doc_cnt()),
|
||||
PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT.c_str());
|
||||
}
|
||||
}
|
||||
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
|
||||
LOG_ERROR("Cannot add vector while dumping index");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
|
||||
ctx->reset_query(query);
|
||||
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
|
||||
ctx->set_provider(provider_);
|
||||
|
||||
if (metric_->support_train()) {
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
ret = metric_->train(query, meta_.dimension());
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw streamer metric train failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
std::string converted_vector;
|
||||
IndexQueryMeta converted_meta;
|
||||
ret = reformer_->convert(query, qmeta, &converted_vector, &converted_meta);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Rabitq hnsw convert failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
level_t level = alg_->get_random_level();
|
||||
ret = entity_.add_vector_with_id(level, id, converted_vector.data());
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw streamer add vector failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = alg_->add_node(id, level, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw steamer add node failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
(*stats_.mutable_added_count())++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Add a vector into index
|
||||
int HnswRabitqStreamer::add_impl(uint64_t pkey, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
IndexStreamer::Context::Pointer &context) {
|
||||
if (!provider_) {
|
||||
LOG_ERROR("Provider is nullptr, cannot add vector");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to HnswRabitqContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
|
||||
if (entity_.doc_cnt() >= docs_hard_limit_) {
|
||||
LOG_ERROR("Current docs %zu exceed [%s]",
|
||||
static_cast<size_t>(entity_.doc_cnt()),
|
||||
PARAM_HNSW_RABITQ_STREAMER_DOCS_HARD_LIMIT.c_str());
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_IndexFull;
|
||||
} else {
|
||||
LOG_WARN("Current docs %zu exceed [%s]",
|
||||
static_cast<size_t>(entity_.doc_cnt()),
|
||||
PARAM_HNSW_RABITQ_STREAMER_DOCS_SOFT_LIMIT.c_str());
|
||||
}
|
||||
}
|
||||
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
|
||||
LOG_ERROR("Cannot add vector while dumping index");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
|
||||
ctx->reset_query(query);
|
||||
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
|
||||
ctx->set_provider(provider_);
|
||||
|
||||
if (metric_->support_train()) {
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
ret = metric_->train(query, meta_.dimension());
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw streamer metric train failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
std::string converted_vector;
|
||||
IndexQueryMeta converted_meta;
|
||||
ret = reformer_->convert(query, qmeta, &converted_vector, &converted_meta);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Rabitq hnsw convert failed, ret=%d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
level_t level = alg_->get_random_level();
|
||||
node_id_t id;
|
||||
ret = entity_.add_vector(level, pkey, converted_vector.data(), &id);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw streamer add vector failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = alg_->add_node(id, level, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw steamer add node failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
(*stats_.mutable_added_count())++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int HnswRabitqStreamer::search_impl(
|
||||
const void *query, const IndexQueryMeta &qmeta,
|
||||
IndexStreamer::Context::Pointer &context) const {
|
||||
return search_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Similarity search
|
||||
int HnswRabitqStreamer::search_impl(
|
||||
const void *query, const IndexQueryMeta &qmeta, uint32_t count,
|
||||
IndexStreamer::Context::Pointer &context) const {
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to HnswRabitqContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (entity_.doc_cnt() <= ctx->get_bruteforce_threshold()) {
|
||||
return search_bf_impl(query, qmeta, count, context);
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
|
||||
ctx->resize_results(count);
|
||||
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
ret = reformer_->transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw searcher transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ret = query_alg_->search(&entity, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw searcher fast search failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void HnswRabitqStreamer::print_debug_info() {
|
||||
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
|
||||
if (entity_.get_key(id) == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
Neighbors neighbours = entity_.get_neighbors(0, id);
|
||||
std::cout << "node: " << id << "; ";
|
||||
if (neighbours.size() == 0) std::cout << std::endl;
|
||||
for (uint32_t i = 0; i < neighbours.size(); ++i) {
|
||||
std::cout << neighbours[i];
|
||||
|
||||
if (i == neighbours.size() - 1) {
|
||||
std::cout << std::endl;
|
||||
} else {
|
||||
std::cout << ", ";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// entity_.print_key_map();
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::search_bf_impl(
|
||||
const void *query, const IndexQueryMeta &qmeta,
|
||||
IndexStreamer::Context::Pointer &context) const {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::search_bf_impl(
|
||||
const void *query, const IndexQueryMeta &qmeta, uint32_t count,
|
||||
IndexStreamer::Context::Pointer &context) const {
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to HnswRabitqContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
|
||||
ctx->resize_results(count);
|
||||
|
||||
if (ctx->group_by_search()) {
|
||||
if (!ctx->group_by().is_valid()) {
|
||||
LOG_ERROR("Invalid group-by function");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
|
||||
return ctx->group_by()(entity_.get_key(id));
|
||||
};
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
ret = reformer_->transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw rabitq streamer transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ctx->group_topk_heaps().clear();
|
||||
|
||||
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
|
||||
if (entity_.get_key(id) == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
|
||||
EstimateRecord dist;
|
||||
query_alg_->get_full_est(id, dist, entity);
|
||||
|
||||
std::string group_id = group_by(id);
|
||||
|
||||
auto &topk_heap = ctx->group_topk_heaps()[group_id];
|
||||
if (topk_heap.empty()) {
|
||||
topk_heap.limit(ctx->group_topk());
|
||||
}
|
||||
topk_heap.emplace_back(id, dist);
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
} else {
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
ret = reformer_->transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw rabitq streamer transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ctx->topk_heap().clear();
|
||||
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
|
||||
if (entity_.get_key(id) == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
|
||||
EstimateRecord dist;
|
||||
query_alg_->get_full_est(id, dist, entity);
|
||||
ctx->topk_heap().emplace(id, dist);
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamer::search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(p_keys.size() != count)) {
|
||||
LOG_ERROR("The size of p_keys is not equal to count");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
HnswRabitqContext *ctx = dynamic_cast<HnswRabitqContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to HnswRabitqContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
//! context is created by another searcher or streamer
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
|
||||
ctx->resize_results(count);
|
||||
|
||||
if (ctx->group_by_search()) {
|
||||
if (!ctx->group_by().is_valid()) {
|
||||
LOG_ERROR("Invalid group-by function");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
|
||||
return ctx->group_by()(entity_.get_key(id));
|
||||
};
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
ret = reformer_->transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw rabitq streamer transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ctx->group_topk_heaps().clear();
|
||||
|
||||
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
|
||||
uint64_t pk = p_keys[q][idx];
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
|
||||
node_id_t id = entity_.get_id(pk);
|
||||
if (id != kInvalidNodeId) {
|
||||
EstimateRecord dist;
|
||||
query_alg_->get_full_est(id, dist, entity);
|
||||
std::string group_id = group_by(id);
|
||||
|
||||
auto &topk_heap = ctx->group_topk_heaps()[group_id];
|
||||
if (topk_heap.empty()) {
|
||||
topk_heap.limit(ctx->group_topk());
|
||||
}
|
||||
topk_heap.emplace_back(id, dist);
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
} else {
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
HnswRabitqQueryEntity entity;
|
||||
ret = reformer_->transform_to_entity(query, &entity);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw rabitq streamer transform failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->reset_query(query);
|
||||
ctx->topk_heap().clear();
|
||||
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
|
||||
key_t pk = p_keys[q][idx];
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
|
||||
node_id_t id = entity_.get_id(pk);
|
||||
if (id != kInvalidNodeId) {
|
||||
EstimateRecord dist;
|
||||
query_alg_->get_full_est(id, dist, entity);
|
||||
ctx->topk_heap().emplace(id, dist);
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,249 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
|
||||
#include "zvec/core/framework/index_framework.h"
|
||||
#include "zvec/core/framework/index_provider.h"
|
||||
#include "zvec/core/framework/index_reformer.h"
|
||||
#include "hnsw_rabitq_algorithm.h"
|
||||
#include "hnsw_rabitq_query_algorithm.h"
|
||||
#include "hnsw_rabitq_streamer_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class HnswRabitqStreamer : public IndexStreamer {
|
||||
public:
|
||||
using ContextPointer = IndexStreamer::Context::Pointer;
|
||||
|
||||
HnswRabitqStreamer();
|
||||
explicit HnswRabitqStreamer(IndexProvider::Pointer provider,
|
||||
RabitqReformer::Pointer reformer = nullptr);
|
||||
virtual ~HnswRabitqStreamer(void);
|
||||
|
||||
HnswRabitqStreamer(const HnswRabitqStreamer &streamer) = delete;
|
||||
HnswRabitqStreamer &operator=(const HnswRabitqStreamer &streamer) = delete;
|
||||
|
||||
void set_provider(IndexProvider::Pointer provider) {
|
||||
provider_ = std::move(provider);
|
||||
}
|
||||
|
||||
void set_reformer(IndexReformer::Pointer reformer) {
|
||||
reformer_ = std::dynamic_pointer_cast<RabitqReformer>(reformer);
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Initialize Streamer
|
||||
virtual int init(const IndexMeta &imeta,
|
||||
const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Streamer
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
//! Create a context
|
||||
virtual Context::Pointer create_context(void) const override;
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Add a vector into index
|
||||
virtual int add_impl(uint64_t pkey, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
//! Add a vector with id into index
|
||||
virtual int add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Fetch vector by key
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
return entity_.get_vector_by_key(key);
|
||||
}
|
||||
|
||||
virtual int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_.get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
//! Fetch vector by id
|
||||
virtual const void *get_vector_by_id(uint32_t id) const override {
|
||||
return entity_.get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_id(
|
||||
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_.get_vector(id, block);
|
||||
}
|
||||
|
||||
//! Open index from file path
|
||||
virtual int open(IndexStorage::Pointer stg) override;
|
||||
|
||||
//! Close file
|
||||
virtual int close(void) override;
|
||||
|
||||
//! flush file
|
||||
virtual int flush(uint64_t checkpoint) override;
|
||||
|
||||
//! Dump index into storage
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
virtual void print_debug_info() override;
|
||||
|
||||
private:
|
||||
inline int check_params(const void *query,
|
||||
const IndexQueryMeta &qmeta) const {
|
||||
if (ailego_unlikely(!query)) {
|
||||
LOG_ERROR("null query");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (ailego_unlikely(qmeta.dimension() != meta_.dimension() ||
|
||||
qmeta.data_type() != meta_.data_type() ||
|
||||
qmeta.element_size() != meta_.element_size())) {
|
||||
LOG_ERROR("Unsupported query meta");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline int check_sparse_count_is_zero(const uint32_t *sparse_count,
|
||||
uint32_t count) const {
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
if (sparse_count[i] != 0)
|
||||
LOG_ERROR("Sparse cout is not empty. Index: %u, Sparse Count: %u", i,
|
||||
sparse_count[i]);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
//! To share ctx across streamer/searcher, we need to update the context for
|
||||
//! current streamer/searcher
|
||||
int update_context(HnswRabitqContext *ctx) const;
|
||||
|
||||
private:
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_OPENED = 2 };
|
||||
class Stats : public IndexStreamer::Stats {
|
||||
public:
|
||||
void clear(void) {
|
||||
set_revision_id(0u);
|
||||
set_loaded_count(0u);
|
||||
set_added_count(0u);
|
||||
set_discarded_count(0u);
|
||||
set_index_size(0u);
|
||||
set_dumped_size(0u);
|
||||
set_check_point(0u);
|
||||
set_create_time(0u);
|
||||
set_update_time(0u);
|
||||
clear_attributes();
|
||||
}
|
||||
};
|
||||
|
||||
HnswRabitqStreamerEntity entity_;
|
||||
HnswRabitqAlgorithm::UPointer alg_;
|
||||
IndexMeta meta_{};
|
||||
IndexMetric::Pointer metric_{};
|
||||
|
||||
IndexMetric::MatrixDistance add_distance_{};
|
||||
IndexMetric::MatrixDistance search_distance_{};
|
||||
|
||||
IndexMetric::MatrixBatchDistance add_batch_distance_{};
|
||||
IndexMetric::MatrixBatchDistance search_batch_distance_{};
|
||||
|
||||
RabitqReformer::Pointer reformer_{}; // RaBitQ reformer
|
||||
HnswRabitqQueryAlgorithm::UPointer query_alg_; // query algorithm
|
||||
// provider_ provides raw vector, which is used to build graph
|
||||
IndexProvider::Pointer provider_{};
|
||||
|
||||
Stats stats_{};
|
||||
std::mutex mutex_{};
|
||||
|
||||
size_t max_index_size_{0UL};
|
||||
size_t chunk_size_{HnswRabitqEntity::kDefaultChunkSize};
|
||||
size_t docs_hard_limit_{HnswRabitqEntity::kDefaultDocsHardLimit};
|
||||
size_t docs_soft_limit_{0UL};
|
||||
uint32_t min_neighbor_cnt_{0u};
|
||||
uint32_t upper_max_neighbor_cnt_{
|
||||
HnswRabitqEntity::kDefaultUpperMaxNeighborCnt};
|
||||
uint32_t l0_max_neighbor_cnt_{HnswRabitqEntity::kDefaultL0MaxNeighborCnt};
|
||||
uint32_t ef_{HnswRabitqEntity::kDefaultEf};
|
||||
uint32_t ef_construction_{HnswRabitqEntity::kDefaultEfConstruction};
|
||||
uint32_t scaling_factor_{HnswRabitqEntity::kDefaultScalingFactor};
|
||||
size_t bruteforce_threshold_{HnswRabitqEntity::kDefaultBruteForceThreshold};
|
||||
size_t max_scan_limit_{HnswRabitqEntity::kDefaultMaxScanLimit};
|
||||
size_t min_scan_limit_{HnswRabitqEntity::kDefaultMinScanLimit};
|
||||
float bf_negative_prob_{HnswRabitqEntity::kDefaultBFNegativeProbability};
|
||||
float max_scan_ratio_{HnswRabitqEntity::kDefaultScanRatio};
|
||||
|
||||
uint32_t magic_{0U};
|
||||
State state_{STATE_INIT};
|
||||
bool bf_enabled_{false};
|
||||
bool check_crc_enabled_{false};
|
||||
bool filter_same_key_{false};
|
||||
bool get_vector_enabled_{false};
|
||||
bool force_padding_topk_enabled_{false};
|
||||
bool use_id_map_{true};
|
||||
|
||||
//! avoid add vector while dumping index
|
||||
ailego::SharedMutex shared_mutex_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,709 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "hnsw_rabitq_streamer_entity.h"
|
||||
#include <ailego/utility/memory_helper.h>
|
||||
|
||||
// #define DEBUG_PRINT
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswRabitqStreamerEntity::HnswRabitqStreamerEntity(IndexStreamer::Stats &stats)
|
||||
: stats_(stats) {}
|
||||
|
||||
HnswRabitqStreamerEntity::~HnswRabitqStreamerEntity() {}
|
||||
|
||||
int HnswRabitqStreamerEntity::init(size_t max_doc_cnt) {
|
||||
if (std::pow(scaling_factor(), kMaxGraphLayers) < max_doc_cnt) {
|
||||
LOG_ERROR("scalingFactor=%zu is too small", scaling_factor());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
broker_ = std::make_shared<HnswRabitqChunkBroker>(stats_);
|
||||
upper_neighbor_index_ = std::make_shared<NIHashMap>();
|
||||
keys_map_lock_ = std::make_shared<ailego::SharedMutex>();
|
||||
keys_map_ = std::make_shared<HashMap<key_t, node_id_t>>();
|
||||
if (!keys_map_ || !upper_neighbor_index_ || !broker_ || !keys_map_lock_) {
|
||||
LOG_ERROR("HnswRabitqStreamerEntity new object failed");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
keys_map_->set_empty_key(kInvalidKey);
|
||||
|
||||
neighbor_size_ = neighbors_size();
|
||||
upper_neighbor_size_ = upper_neighbors_size();
|
||||
|
||||
//! vector + key + level 0 neighbors
|
||||
size_t size = vector_size() + sizeof(key_t) + neighbor_size_;
|
||||
|
||||
size = AlignSize(size);
|
||||
set_node_size(size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::cleanup() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
mutable_header()->clear();
|
||||
chunk_size_ = kDefaultChunkSize;
|
||||
node_index_mask_bits_ = 0U;
|
||||
node_index_mask_ = 0U;
|
||||
node_cnt_per_chunk_ = 0U;
|
||||
neighbor_size_ = 0U;
|
||||
upper_neighbor_size_ = 0U;
|
||||
if (upper_neighbor_index_) {
|
||||
upper_neighbor_index_->cleanup();
|
||||
}
|
||||
if (keys_map_) {
|
||||
keys_map_->clear();
|
||||
}
|
||||
node_chunks_.clear();
|
||||
upper_neighbor_chunks_.clear();
|
||||
filter_same_key_ = false;
|
||||
get_vector_enabled_ = false;
|
||||
broker_.reset();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors) {
|
||||
char buffer[neighbor_size_];
|
||||
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer);
|
||||
hd->neighbor_cnt = neighbors.size();
|
||||
size_t i = 0;
|
||||
for (; i < neighbors.size(); ++i) {
|
||||
hd->neighbors[i] = neighbors[i].first;
|
||||
}
|
||||
|
||||
auto loc = get_neighbor_chunk_loc(level, id);
|
||||
size_t size = reinterpret_cast<char *>(&hd->neighbors[i]) - &buffer[0];
|
||||
size_t ret = loc.first->write(loc.second, hd, size);
|
||||
if (ailego_unlikely(ret != size)) {
|
||||
LOG_ERROR("Write neighbor header failed, ret=%zu", ret);
|
||||
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
const Neighbors HnswRabitqStreamerEntity::get_neighbors(level_t level,
|
||||
node_id_t id) const {
|
||||
Chunk *chunk = nullptr;
|
||||
size_t offset = 0UL;
|
||||
size_t neighbor_size = neighbor_size_;
|
||||
if (level == 0UL) {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
offset =
|
||||
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
|
||||
|
||||
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
|
||||
&node_chunks_);
|
||||
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
|
||||
chunk = node_chunks_[chunk_idx].get();
|
||||
} else {
|
||||
auto p = get_upper_neighbor_chunk_loc(level, id);
|
||||
chunk = upper_neighbor_chunks_[p.first].get();
|
||||
offset = p.second;
|
||||
neighbor_size = upper_neighbor_size_;
|
||||
}
|
||||
|
||||
ailego_assert_with(offset < chunk->data_size(), "invalid chunk offset");
|
||||
IndexStorage::MemoryBlock neighbor_block;
|
||||
size_t size = chunk->read(offset, neighbor_block, neighbor_size);
|
||||
if (ailego_unlikely(size != neighbor_size)) {
|
||||
LOG_ERROR("Read neighbor header failed, ret=%zu", size);
|
||||
return Neighbors();
|
||||
}
|
||||
return Neighbors(std::move(neighbor_block));
|
||||
}
|
||||
|
||||
//! Get vector data by key
|
||||
const void *HnswRabitqStreamerEntity::get_vector(node_id_t id) const {
|
||||
auto loc = get_vector_chunk_loc(id);
|
||||
const void *vec = nullptr;
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
|
||||
"invalid chunk offset");
|
||||
|
||||
size_t read_size = vector_size();
|
||||
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, &vec, read_size);
|
||||
if (ailego_unlikely(ret != read_size)) {
|
||||
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
|
||||
static_cast<size_t>(loc.second), read_size, ret);
|
||||
}
|
||||
|
||||
return vec;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const {
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
auto loc = get_vector_chunk_loc(ids[i]);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
|
||||
"invalid chunk offset");
|
||||
|
||||
size_t read_size = vector_size();
|
||||
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, &vecs[i], read_size);
|
||||
if (ailego_unlikely(ret != read_size)) {
|
||||
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
|
||||
static_cast<size_t>(loc.second), read_size, ret);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::get_vector(
|
||||
const node_id_t id, IndexStorage::MemoryBlock &block) const {
|
||||
auto loc = get_vector_chunk_loc(id);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
|
||||
"invalid chunk offset");
|
||||
|
||||
size_t read_size = vector_size();
|
||||
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, block, read_size);
|
||||
if (ailego_unlikely(ret != read_size)) {
|
||||
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
|
||||
static_cast<size_t>(loc.second), read_size, ret);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
|
||||
vec_blocks.resize(count);
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
auto loc = get_vector_chunk_loc(ids[i]);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
|
||||
"invalid chunk offset");
|
||||
|
||||
size_t read_size = vector_size();
|
||||
|
||||
size_t ret =
|
||||
node_chunks_[loc.first]->read(loc.second, vec_blocks[i], read_size);
|
||||
if (ailego_unlikely(ret != read_size)) {
|
||||
LOG_ERROR("Read vector failed, offset=%zu, read size=%zu, ret=%zu",
|
||||
static_cast<size_t>(loc.second), read_size, ret);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
key_t HnswRabitqStreamerEntity::get_key(node_id_t id) const {
|
||||
if (use_key_info_map_) {
|
||||
auto loc = get_key_chunk_loc(id);
|
||||
IndexStorage::MemoryBlock key_block;
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
|
||||
"invalid chunk offset");
|
||||
size_t ret =
|
||||
node_chunks_[loc.first]->read(loc.second, key_block, sizeof(key_t));
|
||||
if (ailego_unlikely(ret != sizeof(key_t))) {
|
||||
LOG_ERROR("Read vector failed, ret=%zu", ret);
|
||||
return kInvalidKey;
|
||||
}
|
||||
|
||||
return *reinterpret_cast<const key_t *>(key_block.data());
|
||||
} else {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
void HnswRabitqStreamerEntity::add_neighbor(level_t level, node_id_t id,
|
||||
uint32_t size,
|
||||
node_id_t neighbor_id) {
|
||||
auto loc = get_neighbor_chunk_loc(level, id);
|
||||
size_t offset =
|
||||
loc.second + sizeof(NeighborsHeader) + size * sizeof(node_id_t);
|
||||
ailego_assert_with(size < neighbor_cnt(level), "invalid neighbor size");
|
||||
ailego_assert_with(offset < loc.first->data_size(), "invalid chunk offset");
|
||||
size_t ret = loc.first->write(offset, &neighbor_id, sizeof(node_id_t));
|
||||
if (ailego_unlikely(ret != sizeof(node_id_t))) {
|
||||
LOG_ERROR("Write neighbor id failed, ret=%zu", ret);
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t neighbors = size + 1;
|
||||
ret = loc.first->write(loc.second, &neighbors, sizeof(uint32_t));
|
||||
if (ailego_unlikely(ret != sizeof(uint32_t))) {
|
||||
LOG_ERROR("Write neighbor cnt failed, ret=%zu", ret);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::init_chunks(const Chunk::Pointer &header_chunk) {
|
||||
if (header_chunk->data_size() < header_size()) {
|
||||
LOG_ERROR("Invalid header chunk size");
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
IndexStorage::MemoryBlock header_block;
|
||||
size_t size = header_chunk->read(0UL, header_block, header_size());
|
||||
if (ailego_unlikely(size != header_size())) {
|
||||
LOG_ERROR("Read header chunk failed");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
*mutable_header() =
|
||||
*reinterpret_cast<const HNSWHeader *>(header_block.data());
|
||||
|
||||
int ret = check_hnsw_index(&header());
|
||||
if (ret != 0) {
|
||||
broker_->close();
|
||||
return ret;
|
||||
}
|
||||
|
||||
node_chunks_.resize(
|
||||
broker_->get_chunk_cnt(HnswRabitqChunkBroker::CHUNK_TYPE_NODE));
|
||||
for (auto seq = 0UL; seq < node_chunks_.size(); ++seq) {
|
||||
node_chunks_[seq] =
|
||||
broker_->get_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, seq);
|
||||
if (!node_chunks_[seq]) {
|
||||
LOG_ERROR("Missing hnsw streamer data chunk %zu th of %zu", seq,
|
||||
node_chunks_.size());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
}
|
||||
|
||||
upper_neighbor_chunks_.resize(
|
||||
broker_->get_chunk_cnt(HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR));
|
||||
for (auto seq = 0UL; seq < upper_neighbor_chunks_.size(); ++seq) {
|
||||
upper_neighbor_chunks_[seq] = broker_->get_chunk(
|
||||
HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR, seq);
|
||||
if (!upper_neighbor_chunks_[seq]) {
|
||||
LOG_ERROR("Missing hnsw streamer index chunk %zu th of %zu", seq,
|
||||
upper_neighbor_chunks_.size());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::open(IndexStorage::Pointer stg,
|
||||
uint64_t max_index_size, bool check_crc) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
bool huge_page = stg->isHugePage();
|
||||
LOG_DEBUG("huge_page: %d", (int)huge_page);
|
||||
int ret = init_chunk_params(max_index_size, huge_page);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("init_chunk_params failed for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
ret = broker_->open(std::move(stg), max_index_size_, chunk_size_, check_crc);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Open index failed for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
ret = upper_neighbor_index_->init(broker_, upper_neighbor_chunk_size_,
|
||||
scaling_factor(), estimate_doc_capacity(),
|
||||
kUpperHashMemoryInflateRatio);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Init neighbor hash map failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! init header
|
||||
auto header_chunk =
|
||||
broker_->get_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_HEADER,
|
||||
HnswRabitqChunkBroker::kDefaultChunkSeqId);
|
||||
if (!header_chunk) { // open empty index, create one
|
||||
auto p = broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_HEADER,
|
||||
HnswRabitqChunkBroker::kDefaultChunkSeqId,
|
||||
header_size());
|
||||
if (ailego_unlikely(p.first != 0)) {
|
||||
LOG_ERROR("Alloc header chunk failed");
|
||||
return p.first;
|
||||
}
|
||||
size_t size = p.second->write(0UL, &header(), header_size());
|
||||
if (ailego_unlikely(size != header_size())) {
|
||||
LOG_ERROR("Write header chunk failed");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Open an exist hnsw index
|
||||
ret = init_chunks(header_chunk);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
//! total docs including features wrote in index but neighbors may not ready
|
||||
node_id_t total_vecs = 0;
|
||||
if (node_chunks_.size() > 0) {
|
||||
size_t last_idx = node_chunks_.size() - 1;
|
||||
auto last_chunk = node_chunks_[last_idx];
|
||||
if (last_chunk->data_size() % node_size()) {
|
||||
LOG_WARN("The index may broken");
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
total_vecs = last_idx * node_cnt_per_chunk_ +
|
||||
node_chunks_[last_idx]->data_size() / node_size();
|
||||
}
|
||||
|
||||
LOG_INFO(
|
||||
"Open index, l0NeighborCnt=%zu upperNeighborCnt=%zu "
|
||||
"efConstruction=%zu curDocCnt=%u totalVecs=%u maxLevel=%u",
|
||||
l0_neighbor_cnt(), upper_neighbor_cnt(), ef_construction(), doc_cnt(),
|
||||
total_vecs, cur_max_level());
|
||||
//! try to correct the docCnt if index not fully flushed
|
||||
if (doc_cnt() != total_vecs) {
|
||||
LOG_WARN("Index closed abnormally, using totalVecs as curDocCnt");
|
||||
*mutable_doc_cnt() = total_vecs;
|
||||
}
|
||||
if (filter_same_key_ || get_vector_enabled_) {
|
||||
if (use_key_info_map_) {
|
||||
for (node_id_t id = 0U; id < doc_cnt(); ++id) {
|
||||
if (get_key(id) == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
(*keys_map_)[get_key(id)] = id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stats_.set_loaded_count(doc_cnt());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::close() {
|
||||
LOG_DEBUG("close index");
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
flush_header();
|
||||
mutable_header()->reset();
|
||||
upper_neighbor_index_->cleanup();
|
||||
keys_map_->clear();
|
||||
header_.clear();
|
||||
node_chunks_.clear();
|
||||
upper_neighbor_chunks_.clear();
|
||||
|
||||
return broker_->close();
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::flush(uint64_t checkpoint) {
|
||||
LOG_INFO("Flush index, curDocs=%zu", static_cast<size_t>(doc_cnt()));
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
flush_header();
|
||||
int ret = broker_->flush(checkpoint);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::dump(const IndexDumper::Pointer &dumper) {
|
||||
LOG_INFO("Dump index, curDocs=%zu", static_cast<size_t>(doc_cnt()));
|
||||
|
||||
//! sort by keys, to support get_vector by key in searcher
|
||||
std::vector<key_t> keys(doc_cnt());
|
||||
for (node_id_t i = 0; i < doc_cnt(); ++i) {
|
||||
keys[i] = get_key(i);
|
||||
}
|
||||
|
||||
//! dump neighbors
|
||||
auto get_level = [&](node_id_t id) {
|
||||
auto it = upper_neighbor_index_->find(id);
|
||||
if (it == upper_neighbor_index_->end()) {
|
||||
return 0U;
|
||||
};
|
||||
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
|
||||
return meta->level;
|
||||
};
|
||||
auto ret = dump_segments(dumper, keys.data(), get_level);
|
||||
if (ailego_unlikely(ret < 0)) {
|
||||
return ret;
|
||||
}
|
||||
*stats_.mutable_dumped_size() += ret;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::check_hnsw_index(const HNSWHeader *hd) const {
|
||||
if (l0_neighbor_cnt() != hd->l0_neighbor_cnt() ||
|
||||
upper_neighbor_cnt() != hd->upper_neighbor_cnt()) {
|
||||
LOG_ERROR("Param neighbor cnt: %zu:%zu mismatch index previous %zu:%zu",
|
||||
l0_neighbor_cnt(), upper_neighbor_cnt(), hd->l0_neighbor_cnt(),
|
||||
hd->upper_neighbor_cnt());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
if (vector_size() != hd->vector_size()) {
|
||||
LOG_ERROR("vector size %zu mismatch index previous %zu", vector_size(),
|
||||
hd->vector_size());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
if (ef_construction() != hd->ef_construction()) {
|
||||
LOG_WARN("Param efConstruction %zu mismatch index previous %zu",
|
||||
ef_construction(), hd->ef_construction());
|
||||
}
|
||||
if (scaling_factor() != hd->scaling_factor()) {
|
||||
LOG_WARN("Param scalingFactor %zu mismatch index previous %zu",
|
||||
scaling_factor(), hd->scaling_factor());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
if (prune_cnt() != hd->neighbor_prune_cnt()) {
|
||||
LOG_WARN("Param pruneCnt %zu mismatch index previous %zu", prune_cnt(),
|
||||
hd->neighbor_prune_cnt());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
if ((hd->entry_point() != kInvalidNodeId &&
|
||||
hd->entry_point() >= hd->doc_cnt()) ||
|
||||
(hd->entry_point() == kInvalidNodeId && hd->doc_cnt() > 0U)) {
|
||||
LOG_WARN("Invalid entryPoint %zu, docCnt %zu",
|
||||
static_cast<size_t>(hd->entry_point()),
|
||||
static_cast<size_t>(hd->doc_cnt()));
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (hd->entry_point() == kInvalidNodeId &&
|
||||
broker_->get_chunk_cnt(HnswRabitqChunkBroker::CHUNK_TYPE_NODE) > 0) {
|
||||
LOG_WARN("The index is broken, maybe it haven't flush");
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::add_vector(level_t level, key_t key,
|
||||
const void *vec, node_id_t *id) {
|
||||
Chunk::Pointer node_chunk;
|
||||
size_t chunk_offset = -1UL;
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
// duplicate check
|
||||
if (ailego_unlikely(filter_same_key_ && get_id(key) != kInvalidNodeId)) {
|
||||
LOG_WARN("Try to add duplicate key, ignore it");
|
||||
return IndexError_Duplicate;
|
||||
}
|
||||
|
||||
node_id_t local_id = static_cast<node_id_t>(doc_cnt());
|
||||
uint32_t chunk_index = node_chunks_.size() - 1U;
|
||||
if (chunk_index == -1U ||
|
||||
(node_chunks_[chunk_index]->data_size() >=
|
||||
node_cnt_per_chunk_ * node_size())) { // no space left and need to alloc
|
||||
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
|
||||
LOG_ERROR("add vector failed for no memory quota");
|
||||
return IndexError_IndexFull;
|
||||
}
|
||||
chunk_index++;
|
||||
auto p = broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_NODE,
|
||||
chunk_index, chunk_size_);
|
||||
if (ailego_unlikely(p.first != 0)) {
|
||||
LOG_ERROR("Alloc data chunk failed");
|
||||
return p.first;
|
||||
}
|
||||
node_chunk = p.second;
|
||||
chunk_offset = 0UL;
|
||||
node_chunks_.emplace_back(node_chunk);
|
||||
} else {
|
||||
node_chunk = node_chunks_[chunk_index];
|
||||
chunk_offset = node_chunk->data_size();
|
||||
}
|
||||
|
||||
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
|
||||
if (ailego_unlikely(size != vector_size())) {
|
||||
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
|
||||
if (ailego_unlikely(size != sizeof(key_t))) {
|
||||
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
//! level 0 neighbors is inited to zero by default
|
||||
|
||||
int ret = add_upper_neighbor(level, local_id);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
chunk_offset += node_size();
|
||||
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
|
||||
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
if (filter_same_key_ || get_vector_enabled_) {
|
||||
if (use_key_info_map_) {
|
||||
keys_map_lock_->lock();
|
||||
(*keys_map_)[key] = local_id;
|
||||
keys_map_lock_->unlock();
|
||||
}
|
||||
}
|
||||
|
||||
*mutable_doc_cnt() += 1;
|
||||
broker_->mark_dirty();
|
||||
*id = local_id;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswRabitqStreamerEntity::add_vector_with_id(level_t level, node_id_t id,
|
||||
const void *vec) {
|
||||
Chunk::Pointer node_chunk;
|
||||
size_t chunk_offset = -1UL;
|
||||
key_t key = id;
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
|
||||
// duplicate check
|
||||
if (ailego_unlikely(filter_same_key_ && get_id(key) != kInvalidNodeId)) {
|
||||
LOG_WARN("Try to add duplicate key, ignore it");
|
||||
return IndexError_Duplicate;
|
||||
}
|
||||
|
||||
// set node_chunk & chunk_offset if succeed
|
||||
auto func_get_node_chunk_and_offset = [&](node_id_t node_id) -> int {
|
||||
uint32_t chunk_index = node_id >> node_index_mask_bits_;
|
||||
ailego_assert_with(chunk_index <= node_chunks_.size(), "invalid chunk idx");
|
||||
// belongs to next chunk
|
||||
if (chunk_index == node_chunks_.size()) {
|
||||
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
|
||||
LOG_ERROR("add vector failed for no memory quota");
|
||||
return IndexError_IndexFull;
|
||||
}
|
||||
auto p = broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_NODE,
|
||||
chunk_index, chunk_size_);
|
||||
if (ailego_unlikely(p.first != 0)) {
|
||||
LOG_ERROR("Alloc data chunk failed");
|
||||
return p.first;
|
||||
}
|
||||
node_chunk = p.second;
|
||||
node_chunks_.emplace_back(node_chunk);
|
||||
}
|
||||
|
||||
node_chunk = node_chunks_[chunk_index];
|
||||
chunk_offset = (node_id & node_index_mask_) * node_size();
|
||||
return 0;
|
||||
};
|
||||
|
||||
for (size_t start_id = doc_cnt(); start_id < id; ++start_id) {
|
||||
if (auto ret = func_get_node_chunk_and_offset(start_id); ret != 0) {
|
||||
LOG_ERROR("func_get_node_chunk_and_offset failed");
|
||||
return ret;
|
||||
}
|
||||
size_t size = node_chunk->write(chunk_offset + vector_size(), &kInvalidKey,
|
||||
sizeof(key_t));
|
||||
if (ailego_unlikely(size != sizeof(key_t))) {
|
||||
LOG_ERROR("Chunk write key failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
chunk_offset += node_size();
|
||||
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
|
||||
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
|
||||
if (auto ret = func_get_node_chunk_and_offset(id); ret != 0) {
|
||||
LOG_ERROR("func_get_node_chunk_and_offset failed");
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
|
||||
if (ailego_unlikely(size != vector_size())) {
|
||||
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
|
||||
if (ailego_unlikely(size != sizeof(key_t))) {
|
||||
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
//! level 0 neighbors is inited to zero by default
|
||||
|
||||
int ret = add_upper_neighbor(level, id);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (*mutable_doc_cnt() <= id) {
|
||||
*mutable_doc_cnt() = id + 1;
|
||||
chunk_offset += node_size();
|
||||
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
|
||||
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
|
||||
if (filter_same_key_ || get_vector_enabled_) {
|
||||
if (use_key_info_map_) {
|
||||
keys_map_lock_->lock();
|
||||
(*keys_map_)[key] = id;
|
||||
keys_map_lock_->unlock();
|
||||
}
|
||||
}
|
||||
|
||||
broker_->mark_dirty();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void HnswRabitqStreamerEntity::update_ep_and_level(node_id_t ep,
|
||||
level_t level) {
|
||||
HnswRabitqEntity::update_ep_and_level(ep, level);
|
||||
flush_header();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const HnswRabitqEntity::Pointer HnswRabitqStreamerEntity::clone() const {
|
||||
std::vector<Chunk::Pointer> node_chunks;
|
||||
node_chunks.reserve(node_chunks_.size());
|
||||
for (size_t i = 0UL; i < node_chunks_.size(); ++i) {
|
||||
node_chunks.emplace_back(node_chunks_[i]->clone());
|
||||
if (ailego_unlikely(!node_chunks[i])) {
|
||||
LOG_ERROR("HnswRabitqStreamerEntity get chunk failed in clone");
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Chunk::Pointer> upper_neighbor_chunks;
|
||||
upper_neighbor_chunks.reserve(upper_neighbor_chunks_.size());
|
||||
for (size_t i = 0UL; i < upper_neighbor_chunks_.size(); ++i) {
|
||||
upper_neighbor_chunks.emplace_back(upper_neighbor_chunks_[i]->clone());
|
||||
if (ailego_unlikely(!upper_neighbor_chunks[i])) {
|
||||
LOG_ERROR("HnswRabitqStreamerEntity get chunk failed in clone");
|
||||
return HnswRabitqEntity::Pointer();
|
||||
}
|
||||
}
|
||||
|
||||
HnswRabitqStreamerEntity *entity =
|
||||
new (std::nothrow) HnswRabitqStreamerEntity(
|
||||
stats_, header(), chunk_size_, node_index_mask_bits_,
|
||||
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
|
||||
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
|
||||
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_);
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("HnswRabitqStreamerEntity new failed");
|
||||
}
|
||||
return HnswRabitqEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,522 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <sparsehash/dense_hash_map>
|
||||
#include <sparsehash/dense_hash_set>
|
||||
#include <zvec/ailego/container/heap.h>
|
||||
#include "zvec/core/framework/index_framework.h"
|
||||
#include "hnsw_rabitq_chunk.h"
|
||||
#include "hnsw_rabitq_entity.h"
|
||||
#include "hnsw_rabitq_index_hash.h"
|
||||
#include "hnsw_rabitq_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! HnswRabitqStreamerEntity manage vector data, pkey, and node's neighbors
|
||||
class HnswRabitqStreamerEntity : public HnswRabitqEntity {
|
||||
public:
|
||||
//! Cleanup
|
||||
//! return 0 on success, or errCode in failure
|
||||
virtual int cleanup() override;
|
||||
|
||||
//! Make a copy of streamer entity, to support thread-safe operation.
|
||||
//! The segment in container cannot be read concurrenly
|
||||
virtual const HnswRabitqEntity::Pointer clone() const override;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector(node_id_t id) const override;
|
||||
|
||||
//! Get vectors feature data by local ids
|
||||
virtual int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
|
||||
virtual int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
virtual int get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
//! Note: the neighbors cannot be modified, using the following
|
||||
//! method to get WritableNeighbors if want to
|
||||
virtual const Neighbors get_neighbors(level_t level,
|
||||
node_id_t id) const override;
|
||||
|
||||
//! Add vector and key to hnsw entity, and local id will be saved in id
|
||||
virtual int add_vector(level_t level, key_t key, const void *vec,
|
||||
node_id_t *id) override;
|
||||
|
||||
//! Add vector and id to hnsw entity
|
||||
virtual int add_vector_with_id(level_t level, node_id_t id,
|
||||
const void *vec) override;
|
||||
|
||||
virtual int update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors)
|
||||
override;
|
||||
|
||||
//! Append neighbor_id to node id neighbors on level
|
||||
//! Notice: the caller must be ensure the neighbors not full
|
||||
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
|
||||
//! Dump index by dumper
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
|
||||
|
||||
void set_use_key_info_map(bool use_id_map) {
|
||||
use_key_info_map_ = use_id_map;
|
||||
LOG_DEBUG("use_key_info_map_: %d", (int)use_key_info_map_);
|
||||
}
|
||||
|
||||
public:
|
||||
//! Constructor
|
||||
HnswRabitqStreamerEntity(IndexStreamer::Stats &stats);
|
||||
|
||||
//! Destructor
|
||||
~HnswRabitqStreamerEntity();
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector_by_key(key_t key) const override {
|
||||
auto id = get_id(key);
|
||||
return id == kInvalidNodeId ? nullptr : get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_key(
|
||||
const key_t key, IndexStorage::MemoryBlock &block) const override {
|
||||
auto id = get_id(key);
|
||||
if (id != kInvalidNodeId) {
|
||||
return get_vector(id, block);
|
||||
} else {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
}
|
||||
|
||||
//! Init entity
|
||||
int init(size_t max_doc_cnt);
|
||||
|
||||
//! Flush graph entity to disk
|
||||
//! return 0 on success, or errCode in failure
|
||||
int flush(uint64_t checkpoint);
|
||||
|
||||
//! Open entity from storage
|
||||
//! return 0 on success, or errCode in failure
|
||||
int open(IndexStorage::Pointer stg, uint64_t max_index_size, bool check_crc);
|
||||
|
||||
//! Close entity
|
||||
//! return 0 on success, or errCode in failure
|
||||
int close();
|
||||
|
||||
//! Set meta information from entity
|
||||
int set_index_meta(const IndexMeta &meta) const {
|
||||
return IndexHelper::SerializeToStorage(meta, broker_->storage().get());
|
||||
}
|
||||
|
||||
//! Get meta information from entity
|
||||
int get_index_meta(IndexMeta *meta) const {
|
||||
return IndexHelper::DeserializeFromStorage(broker_->storage().get(), meta);
|
||||
}
|
||||
|
||||
//! Set params: chunk size
|
||||
inline void set_chunk_size(size_t val) {
|
||||
chunk_size_ = val;
|
||||
}
|
||||
|
||||
//! Set params
|
||||
inline void set_filter_same_key(bool val) {
|
||||
filter_same_key_ = val;
|
||||
}
|
||||
|
||||
//! Set params
|
||||
inline void set_get_vector(bool val) {
|
||||
get_vector_enabled_ = val;
|
||||
}
|
||||
|
||||
//! Get vector local id by key
|
||||
inline node_id_t get_id(key_t key) const {
|
||||
if (use_key_info_map_) {
|
||||
keys_map_lock_->lock_shared();
|
||||
auto it = keys_map_->find(key);
|
||||
keys_map_lock_->unlock_shared();
|
||||
return it == keys_map_->end() ? kInvalidNodeId : it->second;
|
||||
} else {
|
||||
return key;
|
||||
}
|
||||
}
|
||||
|
||||
void print_key_map() const {
|
||||
std::cout << "key map begins" << std::endl;
|
||||
|
||||
auto iter = keys_map_->begin();
|
||||
while (iter != keys_map_->end()) {
|
||||
std::cout << "key: " << iter->first << ", id: " << iter->second
|
||||
<< std::endl;
|
||||
;
|
||||
iter++;
|
||||
}
|
||||
|
||||
std::cout << "key map ends" << std::endl;
|
||||
}
|
||||
|
||||
//! Get l0 neighbors size
|
||||
inline size_t neighbors_size() const {
|
||||
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
|
||||
}
|
||||
|
||||
//! Get neighbors size for level > 0
|
||||
inline size_t upper_neighbors_size() const {
|
||||
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
union UpperNeighborIndexMeta {
|
||||
struct {
|
||||
uint32_t level : 4;
|
||||
uint32_t index : 28; // index is composite type: chunk idx, and the
|
||||
// N th neighbors in chunk, they two composite
|
||||
// the 28 bits location
|
||||
};
|
||||
uint32_t data;
|
||||
};
|
||||
|
||||
template <class Key, class T>
|
||||
using HashMap = google::dense_hash_map<Key, T, std::hash<Key>>;
|
||||
template <class Key, class T>
|
||||
using HashMapPointer = std::shared_ptr<HashMap<Key, T>>;
|
||||
|
||||
template <class Key>
|
||||
using HashSet = google::dense_hash_set<Key, std::hash<Key>>;
|
||||
template <class Key>
|
||||
using HashSetPointer = std::shared_ptr<HashSet<Key>>;
|
||||
|
||||
//! upper neighbor index hashmap
|
||||
using NIHashMap = HnswIndexHashMap<node_id_t, uint32_t>;
|
||||
using NIHashMapPointer = std::shared_ptr<NIHashMap>;
|
||||
|
||||
//! Private construct, only be called by clone method
|
||||
HnswRabitqStreamerEntity(IndexStreamer::Stats &stats, const HNSWHeader &hd,
|
||||
size_t chunk_size, uint32_t node_index_mask_bits,
|
||||
uint32_t upper_neighbor_mask_bits,
|
||||
bool filter_same_key, bool get_vector_enabled,
|
||||
const NIHashMapPointer &upper_neighbor_index,
|
||||
std::shared_ptr<ailego::SharedMutex> &keys_map_lock,
|
||||
const HashMapPointer<key_t, node_id_t> &keys_map,
|
||||
bool use_key_info_map,
|
||||
std::vector<Chunk::Pointer> &&node_chunks,
|
||||
std::vector<Chunk::Pointer> &&upper_neighbor_chunks,
|
||||
const HnswRabitqChunkBroker::Pointer &broker)
|
||||
: stats_(stats),
|
||||
chunk_size_(chunk_size),
|
||||
node_index_mask_bits_(node_index_mask_bits),
|
||||
node_cnt_per_chunk_(1UL << node_index_mask_bits_),
|
||||
node_index_mask_(node_cnt_per_chunk_ - 1),
|
||||
upper_neighbor_mask_bits_(upper_neighbor_mask_bits),
|
||||
upper_neighbor_mask_((1U << upper_neighbor_mask_bits_) - 1),
|
||||
filter_same_key_(filter_same_key),
|
||||
get_vector_enabled_(get_vector_enabled),
|
||||
use_key_info_map_(use_key_info_map),
|
||||
upper_neighbor_index_(upper_neighbor_index),
|
||||
keys_map_lock_(keys_map_lock),
|
||||
keys_map_(keys_map),
|
||||
node_chunks_(std::move(node_chunks)),
|
||||
upper_neighbor_chunks_(std::move(upper_neighbor_chunks)),
|
||||
broker_(broker) {
|
||||
*mutable_header() = hd;
|
||||
|
||||
neighbor_size_ = neighbors_size();
|
||||
upper_neighbor_size_ = upper_neighbors_size();
|
||||
}
|
||||
|
||||
//! Called only in searching procedure per context, so no need to lock
|
||||
void sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE type, size_t idx,
|
||||
std::vector<Chunk::Pointer> *chunks) const {
|
||||
if (ailego_likely(idx < chunks->size())) {
|
||||
return;
|
||||
}
|
||||
for (size_t i = chunks->size(); i <= idx; ++i) {
|
||||
auto chunk = broker_->get_chunk(type, i);
|
||||
// the storage can ensure get chunk will success after the first get
|
||||
ailego_assert_with(!!chunk, "get chunk failed");
|
||||
chunks->emplace_back(std::move(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
//! return pair: chunk index + chunk offset
|
||||
inline std::pair<uint32_t, uint32_t> get_vector_chunk_loc(
|
||||
node_id_t id) const {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset = (id & node_index_mask_) * node_size();
|
||||
|
||||
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
|
||||
&node_chunks_);
|
||||
return std::make_pair(chunk_idx, offset);
|
||||
}
|
||||
|
||||
//! return pair: chunk index + chunk offset
|
||||
inline std::pair<uint32_t, uint32_t> get_key_chunk_loc(node_id_t id) const {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
|
||||
|
||||
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
|
||||
&node_chunks_);
|
||||
return std::make_pair(chunk_idx, offset);
|
||||
}
|
||||
|
||||
inline std::pair<uint32_t, uint32_t> get_upper_neighbor_chunk_loc(
|
||||
level_t level, node_id_t id) const {
|
||||
auto it = upper_neighbor_index_->find(id);
|
||||
ailego_assert_abort(it != upper_neighbor_index_->end(),
|
||||
"Get upper neighbor header failed");
|
||||
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
|
||||
uint32_t chunk_idx = (meta->index) >> upper_neighbor_mask_bits_;
|
||||
uint32_t offset = (((meta->index) & upper_neighbor_mask_) + level - 1) *
|
||||
upper_neighbor_size_;
|
||||
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR, chunk_idx,
|
||||
&upper_neighbor_chunks_);
|
||||
ailego_assert_abort(chunk_idx < upper_neighbor_chunks_.size(),
|
||||
"invalid chunk idx");
|
||||
ailego_assert_abort(offset < upper_neighbor_chunks_[chunk_idx]->data_size(),
|
||||
"invalid chunk offset");
|
||||
return std::make_pair(chunk_idx, offset);
|
||||
}
|
||||
|
||||
//! return pair: chunk + chunk offset
|
||||
inline std::pair<Chunk *, size_t> get_neighbor_chunk_loc(level_t level,
|
||||
node_id_t id) const {
|
||||
if (level == 0UL) {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset =
|
||||
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
|
||||
|
||||
sync_chunks(HnswRabitqChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
|
||||
&node_chunks_);
|
||||
ailego_assert_abort(chunk_idx < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_abort(offset < node_chunks_[chunk_idx]->data_size(),
|
||||
"invalid chunk offset");
|
||||
return std::make_pair(node_chunks_[chunk_idx].get(), offset);
|
||||
} else {
|
||||
auto p = get_upper_neighbor_chunk_loc(level, id);
|
||||
return std::make_pair(upper_neighbor_chunks_[p.first].get(), p.second);
|
||||
}
|
||||
}
|
||||
|
||||
//! Chunk hnsw index valid
|
||||
int check_hnsw_index(const HNSWHeader *hd) const;
|
||||
|
||||
size_t get_total_upper_neighbors_size(level_t level) const {
|
||||
return level * upper_neighbor_size_;
|
||||
}
|
||||
|
||||
//! Add upper neighbor header and reserve space for upper neighbor
|
||||
int add_upper_neighbor(level_t level, node_id_t id) {
|
||||
if (level == 0) {
|
||||
return 0;
|
||||
}
|
||||
Chunk::Pointer chunk;
|
||||
uint64_t chunk_offset = -1UL;
|
||||
size_t neighbors_size = get_total_upper_neighbors_size(level);
|
||||
uint64_t chunk_index = upper_neighbor_chunks_.size() - 1UL;
|
||||
if (chunk_index == -1UL ||
|
||||
(upper_neighbor_chunks_[chunk_index]->padding_size() <
|
||||
neighbors_size)) { // no space left and need to alloc
|
||||
chunk_index++;
|
||||
if (ailego_unlikely(upper_neighbor_chunks_.capacity() ==
|
||||
upper_neighbor_chunks_.size())) {
|
||||
LOG_ERROR("add upper neighbor failed for no memory quota");
|
||||
return IndexError_IndexFull;
|
||||
}
|
||||
auto p =
|
||||
broker_->alloc_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR,
|
||||
chunk_index, upper_neighbor_chunk_size_);
|
||||
if (ailego_unlikely(p.first != 0)) {
|
||||
LOG_ERROR("Alloc data chunk failed");
|
||||
return p.first;
|
||||
}
|
||||
chunk = p.second;
|
||||
chunk_offset = 0UL;
|
||||
upper_neighbor_chunks_.emplace_back(chunk);
|
||||
} else {
|
||||
chunk = upper_neighbor_chunks_[chunk_index];
|
||||
chunk_offset = chunk->data_size();
|
||||
}
|
||||
ailego_assert_with((size_t)level < kMaxGraphLayers, "invalid level");
|
||||
ailego_assert_with(chunk_offset % upper_neighbor_size_ == 0,
|
||||
"invalid offset");
|
||||
ailego_assert_with((chunk_offset / upper_neighbor_size_) <
|
||||
(1U << upper_neighbor_mask_bits_),
|
||||
"invalid offset");
|
||||
ailego_assert_with(chunk_index < (1U << (28 - upper_neighbor_mask_bits_)),
|
||||
"invalid chunk index");
|
||||
UpperNeighborIndexMeta meta;
|
||||
meta.level = level;
|
||||
meta.index = (chunk_index << upper_neighbor_mask_bits_) |
|
||||
(chunk_offset / upper_neighbor_size_);
|
||||
chunk_offset += upper_neighbor_size_ * level;
|
||||
if (ailego_unlikely(!upper_neighbor_index_->insert(id, meta.data))) {
|
||||
LOG_ERROR("HashMap insert value failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(chunk->resize(chunk_offset) != chunk_offset)) {
|
||||
LOG_ERROR("Chunk resize to %zu failed", (size_t)chunk_offset);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t estimate_doc_capacity() const {
|
||||
return node_chunks_.capacity() * node_cnt_per_chunk_;
|
||||
}
|
||||
|
||||
int init_chunk_params(size_t max_index_size, bool huge_page) {
|
||||
node_cnt_per_chunk_ = std::max<uint32_t>(1, chunk_size_ / node_size());
|
||||
//! align node cnt per chunk to pow of 2
|
||||
node_index_mask_bits_ = std::ceil(std::log2(node_cnt_per_chunk_));
|
||||
node_cnt_per_chunk_ = 1UL << node_index_mask_bits_;
|
||||
if (huge_page) {
|
||||
chunk_size_ = AlignHugePageSize(node_cnt_per_chunk_ * node_size());
|
||||
} else {
|
||||
chunk_size_ = AlignPageSize(node_cnt_per_chunk_ * node_size());
|
||||
}
|
||||
node_index_mask_ = node_cnt_per_chunk_ - 1;
|
||||
|
||||
if (max_index_size == 0UL) {
|
||||
max_index_size_ = chunk_size_ * kDefaultMaxChunkCnt;
|
||||
} else {
|
||||
max_index_size_ = max_index_size;
|
||||
}
|
||||
|
||||
//! To get a balanced upper neighbor chunk size.
|
||||
//! If the upper chunk size is equal to node chunk size, it may waste
|
||||
//! upper neighbor chunk space; if the upper neighbor chunk size is too
|
||||
//! small, the will need large upper neighbor chunks index space. So to
|
||||
//! get a balanced ratio be sqrt of the node/neighbor size ratio
|
||||
float ratio =
|
||||
std::sqrt(node_size() * scaling_factor() * 1.0f / upper_neighbor_size_);
|
||||
if (huge_page) {
|
||||
upper_neighbor_chunk_size_ = AlignHugePageSize(
|
||||
std::max(get_total_upper_neighbors_size(kMaxGraphLayers),
|
||||
static_cast<size_t>(chunk_size_ / ratio)));
|
||||
} else {
|
||||
upper_neighbor_chunk_size_ = AlignPageSize(
|
||||
std::max(get_total_upper_neighbors_size(kMaxGraphLayers),
|
||||
static_cast<size_t>(chunk_size_ / ratio)));
|
||||
}
|
||||
upper_neighbor_mask_bits_ =
|
||||
std::ceil(std::log2(upper_neighbor_chunk_size_ / upper_neighbor_size_));
|
||||
upper_neighbor_mask_ = (1 << upper_neighbor_mask_bits_) - 1;
|
||||
|
||||
size_t max_node_chunk_cnt = std::ceil(max_index_size_ / chunk_size_);
|
||||
size_t max_upper_chunk_cnt = std::ceil(
|
||||
(max_node_chunk_cnt * node_cnt_per_chunk_ * 1.0f / scaling_factor()) /
|
||||
(upper_neighbor_chunk_size_ / upper_neighbor_size_));
|
||||
max_upper_chunk_cnt =
|
||||
max_upper_chunk_cnt + std::ceil(max_upper_chunk_cnt / scaling_factor());
|
||||
|
||||
//! reserve space to avoid memmove in chunks vector emplace chunk, so
|
||||
//! as to lock-free in reading chunk
|
||||
node_chunks_.reserve(max_node_chunk_cnt);
|
||||
upper_neighbor_chunks_.reserve(max_upper_chunk_cnt);
|
||||
|
||||
LOG_DEBUG(
|
||||
"Settings: nodeSize=%zu chunkSize=%u upperNeighborSize=%u "
|
||||
"upperNeighborChunkSize=%u "
|
||||
"nodeCntPerChunk=%u maxChunkCnt=%zu maxNeighborChunkCnt=%zu "
|
||||
"maxIndexSize=%zu ratio=%.3f",
|
||||
node_size(), chunk_size_, upper_neighbor_size_,
|
||||
upper_neighbor_chunk_size_, node_cnt_per_chunk_, max_node_chunk_cnt,
|
||||
max_upper_chunk_cnt, max_index_size_, ratio);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Init node chunk and neighbor chunks
|
||||
int init_chunks(const Chunk::Pointer &header_chunk);
|
||||
|
||||
int flush_header(void) {
|
||||
if (!broker_->dirty()) {
|
||||
// do not need to flush
|
||||
return 0;
|
||||
}
|
||||
auto header_chunk =
|
||||
broker_->get_chunk(HnswRabitqChunkBroker::CHUNK_TYPE_HEADER,
|
||||
HnswRabitqChunkBroker::kDefaultChunkSeqId);
|
||||
if (ailego_unlikely(!header_chunk)) {
|
||||
LOG_ERROR("get header chunk failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
size_t size = header_chunk->write(0UL, &header(), header_size());
|
||||
if (ailego_unlikely(size != header_size())) {
|
||||
LOG_ERROR("Write header chunk failed");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
HnswRabitqStreamerEntity(const HnswRabitqStreamerEntity &) = delete;
|
||||
HnswRabitqStreamerEntity &operator=(const HnswRabitqStreamerEntity &) =
|
||||
delete;
|
||||
static constexpr uint64_t kUpperHashMemoryInflateRatio = 2.0f;
|
||||
|
||||
private:
|
||||
IndexStreamer::Stats &stats_;
|
||||
HNSWHeader header_{};
|
||||
std::mutex mutex_{};
|
||||
size_t max_index_size_{0UL};
|
||||
uint32_t chunk_size_{kDefaultChunkSize};
|
||||
uint32_t upper_neighbor_chunk_size_{kDefaultChunkSize};
|
||||
uint32_t node_index_mask_bits_{0U};
|
||||
uint32_t node_cnt_per_chunk_{0U};
|
||||
uint32_t node_index_mask_{0U};
|
||||
uint32_t neighbor_size_{0U};
|
||||
uint32_t upper_neighbor_size_{0U};
|
||||
//! UpperNeighborIndex.index composite chunkIdx and offset in chunk by the
|
||||
//! following mask
|
||||
uint32_t upper_neighbor_mask_bits_{0U};
|
||||
uint32_t upper_neighbor_mask_{0U};
|
||||
bool filter_same_key_{false};
|
||||
bool get_vector_enabled_{false};
|
||||
bool use_key_info_map_{true};
|
||||
|
||||
NIHashMapPointer upper_neighbor_index_{};
|
||||
|
||||
mutable std::shared_ptr<ailego::SharedMutex> keys_map_lock_{};
|
||||
HashMapPointer<key_t, node_id_t> keys_map_{};
|
||||
|
||||
//! the chunks will be changed in searcher, so need mutable
|
||||
//! data chunk include: vector, key, level 0 neighbors
|
||||
mutable std::vector<Chunk::Pointer> node_chunks_{};
|
||||
|
||||
//! upper neighbor chunk inlude: UpperNeighborHeader + (1~level) neighbors
|
||||
mutable std::vector<Chunk::Pointer> upper_neighbor_chunks_{};
|
||||
|
||||
HnswRabitqChunkBroker::Pointer broker_{}; // chunk broker
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,315 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "rabitq_converter.h"
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <rabitqlib/utils/rotator.hpp>
|
||||
#include <zvec/ailego/container/params.h>
|
||||
#include <zvec/ailego/parallel/thread_pool.h>
|
||||
#include <zvec/ailego/utility/string_helper.h>
|
||||
#include "ailego/pattern/defer.h"
|
||||
#include "algorithm/hnsw_rabitq/rabitq_reformer.h"
|
||||
#include "zvec/core/framework/index_cluster.h"
|
||||
#include "zvec/core/framework/index_error.h"
|
||||
#include "zvec/core/framework/index_factory.h"
|
||||
#include "zvec/core/framework/index_features.h"
|
||||
#include "zvec/core/framework/index_holder.h"
|
||||
#include "zvec/core/framework/index_memory.h"
|
||||
#include "zvec/core/framework/index_meta.h"
|
||||
#include "rabitq_params.h"
|
||||
#include "rabitq_utils.h"
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define strncasecmp _strnicmp
|
||||
#endif
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
RabitqConverter::~RabitqConverter() {
|
||||
this->cleanup();
|
||||
}
|
||||
|
||||
int RabitqConverter::init(const IndexMeta &meta, const ailego::Params ¶ms) {
|
||||
// Copy meta and ensure it has metric information
|
||||
meta_ = meta;
|
||||
dimension_ = meta.dimension();
|
||||
|
||||
if (meta_.metric_name().empty()) {
|
||||
LOG_ERROR("Meta metric is empty");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
// Round up dimension to multiple of 64
|
||||
padded_dim_ = ((dimension_ + 63) / 64) * 64;
|
||||
|
||||
// Get RaBitQ parameters with defaults
|
||||
uint32_t total_bits = 0;
|
||||
params.get(PARAM_RABITQ_TOTAL_BITS, &total_bits);
|
||||
if (total_bits == 0) {
|
||||
total_bits = kDefaultRabitqTotalBits;
|
||||
}
|
||||
if (total_bits < 1 || total_bits > 9) {
|
||||
LOG_ERROR("Invalid total_bits: %zu, must be in [1, 9]", (size_t)total_bits);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
ex_bits_ = total_bits - 1;
|
||||
|
||||
params.get(PARAM_RABITQ_NUM_CLUSTERS, &num_clusters_);
|
||||
if (num_clusters_ == 0) {
|
||||
num_clusters_ = kDefaultNumClusters;
|
||||
}
|
||||
|
||||
if (ex_bits_ > 8) {
|
||||
LOG_ERROR("Invalid ex_bits: %zu, must be <= 8", ex_bits_);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (meta.data_type() != IndexMeta::DataType::DT_FP32) {
|
||||
LOG_ERROR("RaBitQ only supports FP32 data type");
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
params.get(PARAM_RABITQ_SAMPLE_COUNT, &sample_count_);
|
||||
|
||||
std::string rotator_type_str;
|
||||
params.get(PARAM_RABITQ_ROTATOR_TYPE, &rotator_type_str);
|
||||
if (rotator_type_str.empty()) {
|
||||
rotator_type_ = rabitqlib::RotatorType::FhtKacRotator;
|
||||
} else if (strncasecmp(rotator_type_str.c_str(), "fht", 3) == 0) {
|
||||
rotator_type_ = rabitqlib::RotatorType::FhtKacRotator;
|
||||
} else if (strncasecmp(rotator_type_str.c_str(), "matrix", 6) == 0) {
|
||||
rotator_type_ = rabitqlib::RotatorType::MatrixRotator;
|
||||
} else {
|
||||
LOG_ERROR("Invalid rotator_type: %s", rotator_type_str.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
// Create rotator
|
||||
rotator_.reset(
|
||||
rabitqlib::choose_rotator<float>(dimension_, rotator_type_, padded_dim_));
|
||||
|
||||
LOG_INFO(
|
||||
"RabitqConverter initialized: dim=%zu, padded_dim=%zu, "
|
||||
"num_clusters=%zu, ex_bits=%zu, rotator_type=%d[%s] sample_count[%zu]",
|
||||
dimension_, padded_dim_, num_clusters_, ex_bits_, (int)rotator_type_,
|
||||
rotator_type_str.c_str(), sample_count_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqConverter::cleanup() {
|
||||
centroids_.clear();
|
||||
rotated_centroids_.clear();
|
||||
result_holder_.reset();
|
||||
rotator_.reset();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqConverter::train(IndexHolder::Pointer holder) {
|
||||
if (!holder) {
|
||||
LOG_ERROR("Null holder for training");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
|
||||
size_t vector_count = holder->count();
|
||||
if (vector_count == 0) {
|
||||
LOG_ERROR("No vectors for training");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
// do sampling from all data
|
||||
size_t sample_count = vector_count;
|
||||
if (sample_count_ > 0) {
|
||||
sample_count = std::min(sample_count_, vector_count);
|
||||
}
|
||||
LOG_INFO("Training with %zu vectors from %zu of holder", sample_count,
|
||||
vector_count);
|
||||
auto sampler = std::make_shared<SampleIndexFeatures<CompactIndexFeatures>>(
|
||||
meta_, sample_count);
|
||||
auto iter = holder->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Create iterator error");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
for (; iter->is_valid(); iter->next()) {
|
||||
sampler->emplace(iter->data());
|
||||
}
|
||||
|
||||
// Holder is not needed, cleanup it.
|
||||
holder.reset();
|
||||
|
||||
if (sampler->count() == 0) {
|
||||
LOG_ERROR("Load training data error");
|
||||
return IndexError_InvalidLength;
|
||||
}
|
||||
|
||||
|
||||
// Create KmeansCluster for training centroids
|
||||
auto cluster = IndexFactory::CreateCluster("OptKmeansCluster");
|
||||
if (!cluster) {
|
||||
LOG_ERROR("Failed to create OptKmeansCluster");
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
// Initialize cluster
|
||||
LOG_INFO(
|
||||
"Initializing KmeansCluster with meta: dim=%u, data_type=%d, metric=%s",
|
||||
meta_.dimension(), (int)meta_.data_type(), meta_.metric_name().c_str());
|
||||
ailego::Params cluster_params;
|
||||
int ret = cluster->init(meta_, cluster_params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize KmeansCluster: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = cluster->mount(sampler);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to mount training data: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
cluster->suggest(num_clusters_);
|
||||
|
||||
// Perform clustering
|
||||
IndexCluster::CentroidList cents;
|
||||
// TODO: support specify threads with argument
|
||||
auto threads = std::make_shared<SingleQueueIndexThreads>(0, false);
|
||||
ret = cluster->cluster(threads, cents);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to perform clustering: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (cents.size() != num_clusters_) {
|
||||
LOG_WARN("Expected %zu clusters, got %zu", num_clusters_, cents.size());
|
||||
num_clusters_ = cents.size();
|
||||
}
|
||||
// Extract original centroids (for LinearSeeker query)
|
||||
centroids_.resize(num_clusters_ * dimension_);
|
||||
// Extract rotated centroids (for quantization)
|
||||
rotated_centroids_.resize(num_clusters_ * padded_dim_);
|
||||
for (uint32_t i = 0; i < num_clusters_; ++i) {
|
||||
const float *cent_data = static_cast<const float *>(cents[i].feature());
|
||||
// Save original centroids
|
||||
std::memcpy(¢roids_[i * dimension_], cent_data,
|
||||
dimension_ * sizeof(float));
|
||||
// Save rotated centroids
|
||||
this->rotator_->rotate(cent_data, &rotated_centroids_[i * padded_dim_]);
|
||||
}
|
||||
|
||||
stats_.set_trained_count(sampler->count());
|
||||
stats_.set_trained_costtime(timer.milli_seconds());
|
||||
|
||||
LOG_INFO("Training completed: %zu centroids, cost %zu ms", num_clusters_,
|
||||
static_cast<size_t>(timer.milli_seconds()));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int RabitqConverter::transform(IndexHolder::Pointer holder) {
|
||||
if (!holder) {
|
||||
LOG_ERROR("Null holder for transformation");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (rotated_centroids_.empty()) {
|
||||
LOG_ERROR("Centroids not trained yet");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_ERROR("Not implemented");
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
int RabitqConverter::dump(const IndexDumper::Pointer &dumper) {
|
||||
if (!dumper) {
|
||||
LOG_ERROR("Null dumper");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (rotated_centroids_.empty() || centroids_.empty()) {
|
||||
LOG_ERROR("No centroids to dump");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
ailego::ElapsedTime timer;
|
||||
size_t dumped_size = 0;
|
||||
|
||||
int ret = dump_rabitq_centroids(
|
||||
dumper, dimension_, padded_dim_, ex_bits_, num_clusters_, rotator_type_,
|
||||
rotated_centroids_, centroids_, rotator_, &dumped_size);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
stats_.set_dumped_size(dumped_size);
|
||||
stats_.set_dumped_costtime(timer.milli_seconds());
|
||||
|
||||
LOG_INFO("Dump completed: %zu bytes, cost %zu ms", stats_.dumped_size(),
|
||||
static_cast<size_t>(timer.milli_seconds()));
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqConverter::to_reformer(IndexReformer::Pointer *reformer) {
|
||||
auto memory_dumper = IndexFactory::CreateDumper("MemoryDumper");
|
||||
memory_dumper->init(ailego::Params());
|
||||
std::string file_id = ailego::StringHelper::Concat(
|
||||
"rabitq_converter_", ailego::Monotime::MilliSeconds(), rand());
|
||||
int ret = memory_dumper->create(file_id);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to create memory dumper: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
// Release memory
|
||||
AILEGO_DEFER([&file_id]() { IndexMemory::Instance()->remove(file_id); });
|
||||
ret = this->dump(memory_dumper);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to dump RabitqConverter: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
ret = memory_dumper->close();
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to close memory dumper: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto res = std::make_shared<RabitqReformer>();
|
||||
ailego::Params reformer_params;
|
||||
reformer_params.set(PARAM_RABITQ_METRIC_NAME, meta_.metric_name());
|
||||
ret = res->init(reformer_params);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize RabitqReformer: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
auto memory_storage = IndexFactory::CreateStorage("MemoryReadStorage");
|
||||
ret = memory_storage->open(file_id, false);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to open memory storage: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
ret = res->load(memory_storage);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to load RabitqReformer: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
*reformer = std::move(res);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <rabitqlib/utils/rotator.hpp>
|
||||
#include "zvec/core/framework/index_cluster.h"
|
||||
#include "zvec/core/framework/index_converter.h"
|
||||
#include "zvec/core/framework/index_reformer.h"
|
||||
#include "zvec/core/framework/index_threads.h"
|
||||
#include "rabitq_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class RabitqReformer;
|
||||
|
||||
/*! RaBitQ Converter
|
||||
* Trains KMeans centroids and quantizes vectors using RaBitQ
|
||||
*/
|
||||
class RabitqConverter : public IndexConverter {
|
||||
public:
|
||||
//! Constructor
|
||||
RabitqConverter() = default;
|
||||
|
||||
//! Destructor
|
||||
~RabitqConverter() override;
|
||||
|
||||
//! Initialize Converter
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Converter
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Train the data - perform KMeans clustering
|
||||
int train(IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Transform the data - quantize vectors using RaBitQ
|
||||
int transform(IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Dump centroids and config into storage
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve a holder as result
|
||||
IndexHolder::Pointer result(void) const override {
|
||||
return result_holder_;
|
||||
}
|
||||
|
||||
//! Retrieve Index Meta
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
int to_reformer(IndexReformer::Pointer *reformer) override;
|
||||
|
||||
private:
|
||||
static inline size_t AlignSize(size_t size) {
|
||||
return (size + 0x1F) & (~0x1F);
|
||||
}
|
||||
|
||||
private:
|
||||
IndexMeta meta_;
|
||||
IndexHolder::Pointer result_holder_;
|
||||
Stats stats_;
|
||||
size_t sample_count_{0};
|
||||
|
||||
// RaBitQ parameters
|
||||
size_t num_clusters_{0};
|
||||
size_t ex_bits_{0};
|
||||
size_t dimension_{0};
|
||||
size_t padded_dim_{0};
|
||||
|
||||
// Original centroids: num_clusters * dimension (for LinearSeeker query)
|
||||
std::vector<float> centroids_;
|
||||
// Rotated centroids: num_clusters * padded_dim (for quantization)
|
||||
std::vector<float> rotated_centroids_;
|
||||
|
||||
// Rotator for vector transformation
|
||||
rabitqlib::RotatorType rotator_type_{rabitqlib::RotatorType::FhtKacRotator};
|
||||
std::unique_ptr<rabitqlib::Rotator<float>> rotator_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
// Local metric type enum that mirrors rabitqlib::MetricType,
|
||||
// without exposing rabitqlib headers to consumers of this file.
|
||||
enum class RabitqMetricType {
|
||||
kL2 = 0,
|
||||
kIP = 1,
|
||||
};
|
||||
|
||||
// RaBitQ Converter parameters
|
||||
static const std::string PARAM_RABITQ_NUM_CLUSTERS(
|
||||
"proxima.rabitq.num_clusters");
|
||||
static const std::string PARAM_RABITQ_TOTAL_BITS("proxima.rabitq.total_bits");
|
||||
static const std::string PARAM_RABITQ_METRIC_NAME("proxima.rabitq.metric_name");
|
||||
static const std::string PARAM_RABITQ_ROTATOR_TYPE(
|
||||
"proxima.rabitq.rotator.type");
|
||||
static const std::string PARAM_RABITQ_SAMPLE_COUNT(
|
||||
"proxima.rabitq.sample_count");
|
||||
|
||||
// Default values
|
||||
constexpr size_t kDefaultNumClusters = 16;
|
||||
// 4-bit, 5-bit, and 7-bit quantization typically achieve 90%, 95%, and 99%
|
||||
// recall, respectively—without accessing raw vectors for reranking
|
||||
constexpr size_t kDefaultRabitqTotalBits = 7;
|
||||
|
||||
constexpr int kMinRabitqDimSize = 64;
|
||||
constexpr int kMaxRabitqDimSize = 4095;
|
||||
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,467 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "rabitq_reformer.h"
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <rabitqlib/defines.hpp>
|
||||
#include <rabitqlib/index/query.hpp>
|
||||
#include <rabitqlib/quantization/rabitq.hpp>
|
||||
#include <rabitqlib/utils/rotator.hpp>
|
||||
#include <zvec/ailego/logger/logger.h>
|
||||
#include <zvec/ailego/utility/string_helper.h>
|
||||
#include "core/algorithm/cluster/linear_seeker.h"
|
||||
#include "zvec/core/framework/index_error.h"
|
||||
#include "zvec/core/framework/index_factory.h"
|
||||
#include "zvec/core/framework/index_features.h"
|
||||
#include "zvec/core/framework/index_meta.h"
|
||||
#include "zvec/core/framework/index_storage.h"
|
||||
#include "hnsw_rabitq_query_entity.h"
|
||||
#include "rabitq_converter.h"
|
||||
#include "rabitq_utils.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
// All rabitqlib types are confined to this translation unit via pimpl.
|
||||
struct RabitqReformer::Impl {
|
||||
// RaBitQ parameters
|
||||
size_t num_clusters{0};
|
||||
size_t ex_bits{0};
|
||||
size_t dimension{0};
|
||||
size_t padded_dim{0};
|
||||
size_t size_bin_data{0};
|
||||
size_t size_ex_data{0};
|
||||
bool loaded{false};
|
||||
|
||||
// Original centroids: num_clusters * dimension (for LinearSeeker query)
|
||||
std::vector<float> centroids;
|
||||
// Rotated centroids: num_clusters * padded_dim (for quantization)
|
||||
std::vector<float> rotated_centroids;
|
||||
|
||||
rabitqlib::RotatorType rotator_type{rabitqlib::RotatorType::FhtKacRotator};
|
||||
std::unique_ptr<rabitqlib::Rotator<float>> rotator;
|
||||
rabitqlib::quant::RabitqConfig query_config;
|
||||
rabitqlib::quant::RabitqConfig config;
|
||||
rabitqlib::MetricType metric_type{rabitqlib::METRIC_L2};
|
||||
|
||||
LinearSeeker::Pointer centroid_seeker;
|
||||
CoherentIndexFeatures::Pointer centroid_features;
|
||||
|
||||
// Translate local enum to rabitqlib enum (used only inside this .cc).
|
||||
static rabitqlib::MetricType to_rabitq(RabitqMetricType m) {
|
||||
return m == RabitqMetricType::kIP ? rabitqlib::METRIC_IP
|
||||
: rabitqlib::METRIC_L2;
|
||||
}
|
||||
|
||||
// Translate rabitqlib enum to local enum.
|
||||
static RabitqMetricType from_rabitq(rabitqlib::MetricType m) {
|
||||
return m == rabitqlib::METRIC_IP ? RabitqMetricType::kIP
|
||||
: RabitqMetricType::kL2;
|
||||
}
|
||||
|
||||
int quantize_vector(const float *raw_vector, uint32_t cluster_id,
|
||||
std::string *quantized_data) const;
|
||||
};
|
||||
|
||||
RabitqReformer::RabitqReformer() : impl_(std::make_unique<Impl>()) {}
|
||||
|
||||
RabitqReformer::~RabitqReformer() {
|
||||
this->cleanup();
|
||||
}
|
||||
|
||||
size_t RabitqReformer::num_clusters() const {
|
||||
return impl_->num_clusters;
|
||||
}
|
||||
|
||||
RabitqMetricType RabitqReformer::rabitq_metric_type() const {
|
||||
return Impl::from_rabitq(impl_->metric_type);
|
||||
}
|
||||
|
||||
int RabitqReformer::init(const ailego::Params ¶ms) {
|
||||
std::string metric_name = params.get_as_string(PARAM_RABITQ_METRIC_NAME);
|
||||
if (metric_name == "SquaredEuclidean") {
|
||||
impl_->metric_type = rabitqlib::METRIC_L2;
|
||||
} else if (metric_name == "InnerProduct") {
|
||||
impl_->metric_type = rabitqlib::METRIC_IP;
|
||||
} else if (metric_name == "Cosine") {
|
||||
impl_->metric_type = rabitqlib::METRIC_IP;
|
||||
} else {
|
||||
LOG_ERROR("Unsupported metric name: %s", metric_name.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
LOG_DEBUG("Rabitq reformer init done. metric_name=%s metric_type=%d",
|
||||
metric_name.c_str(), static_cast<int>(impl_->metric_type));
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqReformer::cleanup() {
|
||||
impl_->centroids.clear();
|
||||
impl_->rotated_centroids.clear();
|
||||
impl_->centroid_seeker.reset();
|
||||
impl_->centroid_features.reset();
|
||||
impl_->loaded = false;
|
||||
impl_->rotator.reset();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqReformer::unload() {
|
||||
return this->cleanup();
|
||||
}
|
||||
|
||||
int RabitqReformer::load(IndexStorage::Pointer storage) {
|
||||
if (!storage) {
|
||||
LOG_ERROR("Invalid storage for load");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
auto segment = storage->get(RABITQ_CONVERTER_SEG_ID);
|
||||
if (!segment) {
|
||||
LOG_ERROR("Failed to get segment %s", RABITQ_CONVERTER_SEG_ID.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
|
||||
size_t offset = 0;
|
||||
RabitqConverterHeader header;
|
||||
IndexStorage::MemoryBlock block;
|
||||
size_t size = segment->read(offset, block, sizeof(header));
|
||||
if (size != sizeof(header)) {
|
||||
LOG_ERROR("Failed to read header");
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
memcpy(&header, block.data(), sizeof(header));
|
||||
impl_->dimension = header.dim;
|
||||
impl_->padded_dim = header.padded_dim;
|
||||
impl_->ex_bits = header.ex_bits;
|
||||
impl_->num_clusters = header.num_clusters;
|
||||
impl_->rotator_type =
|
||||
static_cast<rabitqlib::RotatorType>(header.rotator_type);
|
||||
offset += sizeof(header);
|
||||
|
||||
// Read rotated centroids
|
||||
size_t rotated_centroids_size =
|
||||
sizeof(float) * header.num_clusters * header.padded_dim;
|
||||
size = segment->read(offset, block, rotated_centroids_size);
|
||||
if (size != rotated_centroids_size) {
|
||||
LOG_ERROR("Failed to read rotated centroids");
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
impl_->rotated_centroids.resize(header.num_clusters * header.padded_dim);
|
||||
memcpy(impl_->rotated_centroids.data(), block.data(), rotated_centroids_size);
|
||||
offset += size;
|
||||
|
||||
// Read original centroids (for LinearSeeker query)
|
||||
size_t centroids_size = sizeof(float) * header.num_clusters * header.dim;
|
||||
size = segment->read(offset, block, centroids_size);
|
||||
if (size != centroids_size) {
|
||||
LOG_ERROR("Failed to read centroids");
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
impl_->centroids.resize(header.num_clusters * header.dim);
|
||||
memcpy(impl_->centroids.data(), block.data(), centroids_size);
|
||||
offset += size;
|
||||
|
||||
// Read rotator
|
||||
size_t rotator_size = header.rotator_size;
|
||||
size = segment->read(offset, block, rotator_size);
|
||||
if (size != rotator_size) {
|
||||
LOG_ERROR("Failed to read rotator");
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
impl_->rotator.reset(rabitqlib::choose_rotator<float>(
|
||||
impl_->dimension, impl_->rotator_type, impl_->padded_dim));
|
||||
impl_->rotator->load(reinterpret_cast<const char *>(block.data()));
|
||||
offset += size;
|
||||
|
||||
impl_->query_config = rabitqlib::quant::faster_config(
|
||||
impl_->padded_dim, rabitqlib::SplitSingleQuery<float>::kNumBits);
|
||||
impl_->config =
|
||||
rabitqlib::quant::faster_config(impl_->padded_dim, impl_->ex_bits + 1);
|
||||
|
||||
impl_->size_bin_data =
|
||||
rabitqlib::BinDataMap<float>::data_bytes(impl_->padded_dim);
|
||||
impl_->size_ex_data = rabitqlib::ExDataMap<float>::data_bytes(
|
||||
impl_->padded_dim, impl_->ex_bits);
|
||||
|
||||
// Initialize LinearSeeker for centroid search
|
||||
IndexMeta centroid_meta;
|
||||
centroid_meta.set_data_type(IndexMeta::DataType::DT_FP32);
|
||||
centroid_meta.set_dimension(static_cast<uint32_t>(impl_->dimension));
|
||||
// Note:
|
||||
// 1. spherical kmeans is used for InnerProduct and Cosine, so centroids are
|
||||
// normalized.
|
||||
// 2. for Cosine metric, `transform_to_entity` input is normalized, need to
|
||||
// use InnerProduct metric as Cosine metric requires extra dimension which is
|
||||
// unsuitable for centroids.
|
||||
centroid_meta.set_metric(impl_->metric_type == rabitqlib::METRIC_L2
|
||||
? "SquaredEuclidean"
|
||||
: "InnerProduct",
|
||||
0, ailego::Params());
|
||||
|
||||
impl_->centroid_features = std::make_shared<CoherentIndexFeatures>();
|
||||
impl_->centroid_features->mount(centroid_meta, impl_->centroids.data(),
|
||||
impl_->centroids.size() * sizeof(float));
|
||||
|
||||
impl_->centroid_seeker = std::make_shared<LinearSeeker>();
|
||||
int ret = impl_->centroid_seeker->init(centroid_meta);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to init centroid seeker. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
ret = impl_->centroid_seeker->mount(impl_->centroid_features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to mount centroid features. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO(
|
||||
"Rabitq reformer load done. dimension=%zu, padded_dim=%zu, "
|
||||
"ex_bits=%zu, num_clusters=%zu, size_bin_data=%zu, size_ex_data=%zu "
|
||||
"rotator_type=%d",
|
||||
impl_->dimension, impl_->padded_dim, impl_->ex_bits, impl_->num_clusters,
|
||||
impl_->size_bin_data, impl_->size_ex_data, (int)impl_->rotator_type);
|
||||
impl_->loaded = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqReformer::convert(const void *record, const IndexQueryMeta &rmeta,
|
||||
std::string *out, IndexQueryMeta *ometa) const {
|
||||
if (!impl_->loaded) {
|
||||
LOG_ERROR("Centroids not loaded yet");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (!record || !out) {
|
||||
LOG_ERROR("Invalid arguments for convert");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
// input may be transformed, require rmeta.dimension >= dimension
|
||||
if (rmeta.dimension() < impl_->dimension ||
|
||||
rmeta.data_type() != IndexMeta::DataType::DT_FP32) {
|
||||
LOG_ERROR("Invalid record meta: dimension=%zu, data_type=%d",
|
||||
static_cast<size_t>(rmeta.dimension()), (int)rmeta.data_type());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
// Find nearest centroid using LinearSeeker
|
||||
Seeker::Document doc;
|
||||
int ret = impl_->centroid_seeker->seek(
|
||||
record, impl_->dimension * sizeof(float), &doc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to seek centroid. ret[%d]", ret);
|
||||
return ret;
|
||||
}
|
||||
uint32_t cluster_id = doc.index;
|
||||
|
||||
const float *vector = static_cast<const float *>(record);
|
||||
ret = impl_->quantize_vector(vector, cluster_id, out);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to quantize vector");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ometa->set_meta(IndexMeta::DataType::DT_INT8, (uint32_t)out->size());
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqReformer::transform(const void *, const IndexQueryMeta &,
|
||||
std::string *, IndexQueryMeta *) const {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
int RabitqReformer::transform_to_entity(const void *query,
|
||||
HnswRabitqQueryEntity *entity) const {
|
||||
if (!impl_->loaded) {
|
||||
LOG_ERROR("Centroids not loaded yet");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (!query) {
|
||||
LOG_ERROR("Invalid arguments for transform");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
const float *query_vector = static_cast<const float *>(query);
|
||||
|
||||
// Apply rotator
|
||||
entity->rotated_query.resize(impl_->padded_dim);
|
||||
impl_->rotator->rotate(query_vector, entity->rotated_query.data());
|
||||
|
||||
// Quantize query to 4-bit representation
|
||||
entity->query_wrapper = std::make_unique<rabitqlib::SplitSingleQuery<float>>(
|
||||
entity->rotated_query.data(), impl_->padded_dim, impl_->ex_bits,
|
||||
impl_->query_config, impl_->metric_type);
|
||||
|
||||
// Preprocess - get the distance from query to all centroids
|
||||
entity->q_to_centroids.resize(impl_->num_clusters);
|
||||
|
||||
if (impl_->metric_type == rabitqlib::METRIC_L2) {
|
||||
for (size_t i = 0; i < impl_->num_clusters; i++) {
|
||||
entity->q_to_centroids[i] = std::sqrt(rabitqlib::euclidean_sqr(
|
||||
entity->rotated_query.data(),
|
||||
impl_->rotated_centroids.data() + (i * impl_->padded_dim),
|
||||
impl_->padded_dim));
|
||||
}
|
||||
} else if (impl_->metric_type == rabitqlib::METRIC_IP) {
|
||||
entity->q_to_centroids.resize(impl_->num_clusters * 2);
|
||||
// first half as g_add, second half as g_error
|
||||
for (size_t i = 0; i < impl_->num_clusters; i++) {
|
||||
entity->q_to_centroids[i] = rabitqlib::dot_product(
|
||||
entity->rotated_query.data(),
|
||||
impl_->rotated_centroids.data() + (i * impl_->padded_dim),
|
||||
impl_->padded_dim);
|
||||
entity->q_to_centroids[i + impl_->num_clusters] =
|
||||
std::sqrt(rabitqlib::euclidean_sqr(
|
||||
entity->rotated_query.data(),
|
||||
impl_->rotated_centroids.data() + (i * impl_->padded_dim),
|
||||
impl_->padded_dim));
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqReformer::Impl::quantize_vector(const float *raw_vector,
|
||||
uint32_t cluster_id,
|
||||
std::string *quantized_data) const {
|
||||
std::vector<float> rotated_data(padded_dim);
|
||||
rotator->rotate(raw_vector, rotated_data.data());
|
||||
|
||||
// quantized format: cluster_id + bin_data + ex_data
|
||||
quantized_data->resize(sizeof(cluster_id) + size_bin_data + size_ex_data);
|
||||
memcpy(&(*quantized_data)[0], &cluster_id, sizeof(cluster_id));
|
||||
int bin_data_offset = sizeof(cluster_id);
|
||||
int ex_data_offset = bin_data_offset + size_bin_data;
|
||||
rabitqlib::quant::quantize_split_single(
|
||||
rotated_data.data(), rotated_centroids.data() + (cluster_id * padded_dim),
|
||||
padded_dim, ex_bits, &(*quantized_data)[bin_data_offset],
|
||||
&(*quantized_data)[ex_data_offset], metric_type, config);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqReformer::dump(const IndexDumper::Pointer &dumper) {
|
||||
if (!dumper) {
|
||||
LOG_ERROR("Null dumper");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!impl_->loaded || impl_->rotated_centroids.empty() ||
|
||||
impl_->centroids.empty()) {
|
||||
LOG_ERROR("No centroids to dump");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
size_t dumped_size = 0;
|
||||
int ret = dump_rabitq_centroids(
|
||||
dumper, impl_->dimension, impl_->padded_dim, impl_->ex_bits,
|
||||
impl_->num_clusters, impl_->rotator_type, impl_->rotated_centroids,
|
||||
impl_->centroids, impl_->rotator, &dumped_size);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_INFO("RabitqReformer dump completed: %zu bytes", dumped_size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int RabitqReformer::dump(const IndexStorage::Pointer &storage) {
|
||||
if (!storage) {
|
||||
LOG_ERROR("Null storage");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!impl_->loaded || impl_->rotated_centroids.empty() ||
|
||||
impl_->centroids.empty()) {
|
||||
LOG_ERROR("No centroids to dump");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
auto align_size = [](size_t size) -> size_t {
|
||||
return (size + 0x1F) & (~0x1F);
|
||||
};
|
||||
|
||||
size_t header_size = sizeof(RabitqConverterHeader);
|
||||
size_t rotated_centroids_size =
|
||||
impl_->rotated_centroids.size() * sizeof(float);
|
||||
size_t centroids_size = impl_->centroids.size() * sizeof(float);
|
||||
size_t rotator_size = impl_->rotator->dump_bytes();
|
||||
size_t data_size =
|
||||
header_size + rotated_centroids_size + centroids_size + rotator_size;
|
||||
size_t total_size = align_size(data_size);
|
||||
|
||||
int ret = storage->append(RABITQ_CONVERTER_SEG_ID, total_size);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to append segment %s, ret=%d",
|
||||
RABITQ_CONVERTER_SEG_ID.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto segment = storage->get(RABITQ_CONVERTER_SEG_ID);
|
||||
if (!segment) {
|
||||
LOG_ERROR("Failed to get segment %s", RABITQ_CONVERTER_SEG_ID.c_str());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
size_t offset = 0;
|
||||
|
||||
RabitqConverterHeader header;
|
||||
header.dim = static_cast<uint32_t>(impl_->dimension);
|
||||
header.padded_dim = static_cast<uint32_t>(impl_->padded_dim);
|
||||
header.num_clusters = static_cast<uint32_t>(impl_->num_clusters);
|
||||
header.ex_bits = static_cast<uint8_t>(impl_->ex_bits);
|
||||
header.rotator_type = static_cast<uint8_t>(impl_->rotator_type);
|
||||
header.rotator_size = static_cast<uint32_t>(rotator_size);
|
||||
size_t written = segment->write(offset, &header, header_size);
|
||||
if (written != header_size) {
|
||||
LOG_ERROR("Failed to write header: written=%zu, expected=%zu", written,
|
||||
header_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
offset += header_size;
|
||||
|
||||
written = segment->write(offset, impl_->rotated_centroids.data(),
|
||||
rotated_centroids_size);
|
||||
if (written != rotated_centroids_size) {
|
||||
LOG_ERROR("Failed to write rotated centroids: written=%zu, expected=%zu",
|
||||
written, rotated_centroids_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
offset += rotated_centroids_size;
|
||||
|
||||
written = segment->write(offset, impl_->centroids.data(), centroids_size);
|
||||
if (written != centroids_size) {
|
||||
LOG_ERROR("Failed to write centroids: written=%zu, expected=%zu", written,
|
||||
centroids_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
offset += centroids_size;
|
||||
|
||||
std::vector<char> buffer(rotator_size);
|
||||
impl_->rotator->save(buffer.data());
|
||||
written = segment->write(offset, buffer.data(), rotator_size);
|
||||
if (written != rotator_size) {
|
||||
LOG_ERROR("Failed to write rotator data: written=%zu, expected=%zu",
|
||||
written, rotator_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
LOG_INFO("RabitqReformer dump to storage completed: %zu bytes", data_size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "zvec/core/framework/index_dumper.h"
|
||||
#include "zvec/core/framework/index_reformer.h"
|
||||
#include "zvec/core/framework/index_storage.h"
|
||||
#include "rabitq_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class HnswRabitqQueryEntity;
|
||||
|
||||
/*! RaBitQ Reformer
|
||||
* Loads centroids and performs query transformation and vector quantization.
|
||||
*
|
||||
* All rabitqlib types are hidden behind a pimpl to avoid leaking rabitqlib
|
||||
* headers to consumers of this class.
|
||||
*/
|
||||
class RabitqReformer : public IndexReformer {
|
||||
public:
|
||||
typedef std::shared_ptr<RabitqReformer> Pointer;
|
||||
|
||||
RabitqReformer();
|
||||
~RabitqReformer() override;
|
||||
|
||||
// Non-copyable
|
||||
RabitqReformer(const RabitqReformer &) = delete;
|
||||
RabitqReformer &operator=(const RabitqReformer &) = delete;
|
||||
|
||||
int init(const ailego::Params ¶ms) override;
|
||||
int cleanup(void) override;
|
||||
int load(IndexStorage::Pointer storage) override;
|
||||
int unload(void) override;
|
||||
|
||||
// transform() is not implemented for RabitqReformer; use transform_to_entity.
|
||||
int transform(const void *query, const IndexQueryMeta &qmeta,
|
||||
std::string *out, IndexQueryMeta *ometa) const override;
|
||||
|
||||
int convert(const void *record, const IndexQueryMeta &rmeta, std::string *out,
|
||||
IndexQueryMeta *ometa) const override;
|
||||
|
||||
int dump(const IndexDumper::Pointer &dumper);
|
||||
int dump(const IndexStorage::Pointer &storage);
|
||||
|
||||
int transform_to_entity(const void *query,
|
||||
HnswRabitqQueryEntity *entity) const;
|
||||
|
||||
size_t num_clusters() const;
|
||||
RabitqMetricType rabitq_metric_type() const;
|
||||
|
||||
private:
|
||||
struct Impl;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,115 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "rabitq_utils.h"
|
||||
#include <string>
|
||||
#include <zvec/ailego/hash/crc32c.h>
|
||||
#include "zvec/core/framework/index_error.h"
|
||||
#include "zvec/core/framework/index_logger.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int dump_rabitq_centroids(
|
||||
const IndexDumper::Pointer &dumper, size_t dimension, size_t padded_dim,
|
||||
size_t ex_bits, size_t num_clusters, rabitqlib::RotatorType rotator_type,
|
||||
const std::vector<float> &rotated_centroids,
|
||||
const std::vector<float> ¢roids,
|
||||
const std::unique_ptr<rabitqlib::Rotator<float>> &rotator,
|
||||
size_t *out_dumped_size) {
|
||||
auto align_size = [](size_t size) -> size_t {
|
||||
return (size + 0x1F) & (~0x1F);
|
||||
};
|
||||
|
||||
uint32_t crc = 0;
|
||||
size_t dumped_size = 0;
|
||||
|
||||
// Write header
|
||||
RabitqConverterHeader header;
|
||||
header.dim = static_cast<uint32_t>(dimension);
|
||||
header.padded_dim = static_cast<uint32_t>(padded_dim);
|
||||
header.num_clusters = static_cast<uint32_t>(num_clusters);
|
||||
header.ex_bits = static_cast<uint8_t>(ex_bits);
|
||||
header.rotator_type = static_cast<uint8_t>(rotator_type);
|
||||
header.rotator_size = static_cast<uint32_t>(rotator->dump_bytes());
|
||||
size_t size = dumper->write(&header, sizeof(header));
|
||||
if (size != sizeof(header)) {
|
||||
LOG_ERROR("Failed to write header: written=%zu, expected=%zu", size,
|
||||
sizeof(header));
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(&header, sizeof(header), crc);
|
||||
dumped_size += size;
|
||||
|
||||
// Write rotated centroids
|
||||
size = dumper->write(rotated_centroids.data(),
|
||||
rotated_centroids.size() * sizeof(float));
|
||||
if (size != rotated_centroids.size() * sizeof(float)) {
|
||||
LOG_ERROR("Failed to write rotated centroids: written=%zu, expected=%zu",
|
||||
size, rotated_centroids.size() * sizeof(float));
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(rotated_centroids.data(),
|
||||
rotated_centroids.size() * sizeof(float), crc);
|
||||
dumped_size += size;
|
||||
|
||||
// Write original centroids
|
||||
size = dumper->write(centroids.data(), centroids.size() * sizeof(float));
|
||||
if (size != centroids.size() * sizeof(float)) {
|
||||
LOG_ERROR("Failed to write centroids: written=%zu, expected=%zu", size,
|
||||
centroids.size() * sizeof(float));
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(centroids.data(), centroids.size() * sizeof(float),
|
||||
crc);
|
||||
dumped_size += size;
|
||||
|
||||
// Write rotator data
|
||||
std::vector<char> buffer(rotator->dump_bytes());
|
||||
rotator->save(buffer.data());
|
||||
size = dumper->write(buffer.data(), buffer.size());
|
||||
if (size != buffer.size()) {
|
||||
LOG_ERROR("Failed to write rotator data: written=%zu, expected=%zu", size,
|
||||
buffer.size());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(buffer.data(), buffer.size(), crc);
|
||||
dumped_size += size;
|
||||
|
||||
// Write padding
|
||||
size_t padding_size = align_size(dumped_size) - dumped_size;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
int ret =
|
||||
dumper->append(RABITQ_CONVERTER_SEG_ID, dumped_size, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump segment %s meta failed, ret=%d",
|
||||
RABITQ_CONVERTER_SEG_ID.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (out_dumped_size) {
|
||||
*out_dumped_size = dumped_size;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,53 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <rabitqlib/utils/rotator.hpp>
|
||||
#include "zvec/core/framework/index_dumper.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
inline const std::string RABITQ_CONVERTER_SEG_ID{"rabitq.converter"};
|
||||
|
||||
struct RabitqConverterHeader {
|
||||
uint32_t num_clusters;
|
||||
uint32_t dim;
|
||||
uint32_t padded_dim;
|
||||
uint32_t rotator_size;
|
||||
uint8_t ex_bits;
|
||||
uint8_t rotator_type;
|
||||
uint8_t padding[2];
|
||||
uint32_t reserve[3];
|
||||
|
||||
RabitqConverterHeader() {
|
||||
memset(this, 0, sizeof(RabitqConverterHeader));
|
||||
}
|
||||
};
|
||||
static_assert(sizeof(RabitqConverterHeader) % 32 == 0,
|
||||
"RabitqConverterHeader must be aligned with 32 bytes");
|
||||
|
||||
// Common dump implementation for RabitqConverter and RabitqReformer
|
||||
int dump_rabitq_centroids(
|
||||
const IndexDumper::Pointer &dumper, size_t dimension, size_t padded_dim,
|
||||
size_t ex_bits, size_t num_clusters, rabitqlib::RotatorType rotator_type,
|
||||
const std::vector<float> &rotated_centroids,
|
||||
const std::vector<float> ¢roids,
|
||||
const std::unique_ptr<rabitqlib::Rotator<float>> &rotator,
|
||||
size_t *out_dumped_size = nullptr);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
Reference in New Issue
Block a user