feat: refac entity and impl Vamana. (#371)

This commit is contained in:
luoxiaojian
2026-04-30 10:22:25 +08:00
committed by GitHub
parent 005680522b
commit efab064676
64 changed files with 8669 additions and 201 deletions
+1
View File
@@ -7,6 +7,7 @@ cc_directory(flat_sparse)
cc_directory(ivf)
cc_directory(hnsw)
cc_directory(hnsw_sparse)
cc_directory(vamana)
if(RABITQ_SUPPORTED)
message(STATUS "BUILD RABITQ")
cc_directory(hnsw_rabitq)
+65 -56
View File
@@ -12,28 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_algorithm.h"
#include <chrono>
#include <iostream>
#include <vector>
#include <ailego/internal/cpu_features.h>
namespace zvec {
namespace core {
HnswAlgorithm::HnswAlgorithm(HnswEntity &entity)
: entity_(entity),
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
lock_pool_(kLockCnt) {}
int HnswAlgorithm::cleanup() {
return 0;
}
int HnswAlgorithm::add_node(node_id_t id, level_t level, HnswContext *ctx) {
template <typename EntityType>
int HnswAlgorithm<EntityType>::add_node(node_id_t id, level_t level,
HnswContext *ctx) {
spin_lock_.lock();
// std::cout << "id: " << id << ", level: " << level << std::endl;
auto cur_max_level = entity_.cur_max_level();
auto entry_point = entity_.entry_point();
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
@@ -54,7 +41,7 @@ int HnswAlgorithm::add_node(node_id_t id, level_t level, HnswContext *ctx) {
}
level_t cur_level = cur_max_level;
dist_t dist = ctx->dist_calculator()(entry_point);
dist_t dist = ctx->dist_calculator().batch_dist(entry_point);
for (; cur_level > level; --cur_level) {
select_entry_point(cur_level, &entry_point, &dist, ctx);
}
@@ -81,7 +68,8 @@ int HnswAlgorithm::add_node(node_id_t id, level_t level, HnswContext *ctx) {
return 0;
}
int HnswAlgorithm::search(HnswContext *ctx) const {
template <typename EntityType>
int HnswAlgorithm<EntityType>::search(HnswContext *ctx) const {
spin_lock_.lock();
auto maxLevel = entity_.cur_max_level();
auto entry_point = entity_.entry_point();
@@ -107,13 +95,15 @@ int HnswAlgorithm::search(HnswContext *ctx) const {
return 0;
}
//! select_entry_point on hnsw level, ef = 1
void HnswAlgorithm::select_entry_point(level_t level, node_id_t *entry_point,
dist_t *dist, HnswContext *ctx) const {
auto &entity = ctx->get_entity();
template <typename EntityType>
void HnswAlgorithm<EntityType>::select_entry_point(level_t level,
node_id_t *entry_point,
dist_t *dist,
HnswContext *ctx) const {
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
HnswDistCalculator &dc = ctx->dist_calculator();
while (true) {
const Neighbors neighbors = entity.get_neighbors(level, *entry_point);
const auto neighbors = entity.get_neighbors_typed(level, *entry_point);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
@@ -122,8 +112,8 @@ void HnswAlgorithm::select_entry_point(level_t level, node_id_t *entry_point,
break;
}
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
int ret = entity.get_vector(&neighbors[0], size, neighbor_vec_blocks);
std::vector<MemBlockType> neighbor_vec_blocks;
int ret = entity.get_vector_typed(&neighbors[0], size, neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
@@ -159,8 +149,10 @@ void HnswAlgorithm::select_entry_point(level_t level, node_id_t *entry_point,
return;
}
void HnswAlgorithm::add_neighbors(node_id_t id, level_t level,
TopkHeap &topk_heap, HnswContext *ctx) {
template <typename EntityType>
void HnswAlgorithm<EntityType>::add_neighbors(node_id_t id, level_t level,
TopkHeap &topk_heap,
HnswContext *ctx) {
if (ailego_unlikely(topk_heap.size() == 0)) {
return;
}
@@ -178,16 +170,20 @@ void HnswAlgorithm::add_neighbors(node_id_t id, level_t level,
return;
}
void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
dist_t *dist, TopkHeap &topk,
HnswContext *ctx) const {
const auto &entity = ctx->get_entity();
template <typename EntityType>
void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
node_id_t *entry_point,
dist_t *dist, TopkHeap &topk,
HnswContext *ctx) const {
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
HnswDistCalculator &dc = ctx->dist_calculator();
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (ctx->filter().is_valid()) {
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
filter = [&](node_id_t id) {
return ctx->filter()(entity.get_key_typed(id));
};
}
candidates.clear();
@@ -208,7 +204,7 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
}
candidates.pop();
const Neighbors neighbors = entity.get_neighbors(level, main_node);
const auto neighbors = entity.get_neighbors_typed(level, main_node);
ailego_prefetch(neighbors.data);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
@@ -231,8 +227,9 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
continue;
}
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
int ret = entity.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
std::vector<MemBlockType> neighbor_vec_blocks;
int ret =
entity.get_vector_typed(neighbor_ids.data(), size, neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
@@ -278,15 +275,16 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
return;
}
void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
HnswContext *ctx) const {
template <typename EntityType>
void HnswAlgorithm<EntityType>::expand_neighbors_by_group(
TopkHeap &topk, HnswContext *ctx) const {
if (!ctx->group_by().is_valid()) {
return;
}
const auto &entity = ctx->get_entity();
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity.get_key(id));
return ctx->group_by()(entity.get_key_typed(id));
};
// devide into groups
@@ -312,7 +310,9 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (ctx->filter().is_valid()) {
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
filter = [&](node_id_t id) {
return ctx->filter()(entity.get_key_typed(id));
};
}
// refill to get enough groups
@@ -332,7 +332,7 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
node_id_t main_node = top->first;
candidates.pop();
const Neighbors neighbors = entity.get_neighbors(0, main_node);
const auto neighbors = entity.get_neighbors_typed(0, main_node);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_neighbors())++;
}
@@ -354,9 +354,9 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
continue;
}
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
int ret =
entity.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
std::vector<MemBlockType> neighbor_vec_blocks;
int ret = entity.get_vector_typed(neighbor_ids.data(), size,
neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
@@ -364,14 +364,16 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
break;
}
static constexpr node_id_t PREFETCH_STEP = 2;
std::vector<float> dists(size);
std::vector<const void *> neighbor_vecs(size);
for (uint32_t i = 0; i < size; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
}
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
for (uint32_t i = 0; i < size; ++i) {
node_id_t node = neighbor_ids[i];
node_id_t prefetch_id = i + PREFETCH_STEP;
if (prefetch_id < size) {
ailego_prefetch(neighbor_vec_blocks[prefetch_id].data());
}
dist_t cur_dist = dc.dist(neighbor_vec_blocks[i].data());
dist_t cur_dist = dists[i];
if (!filter(node)) {
std::string group_id = group_by(node);
@@ -393,8 +395,10 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
} // end if
}
void HnswAlgorithm::update_neighbors(HnswDistCalculator &dc, node_id_t id,
level_t level, TopkHeap &topk_heap) {
template <typename EntityType>
void HnswAlgorithm<EntityType>::update_neighbors(HnswDistCalculator &dc,
node_id_t id, level_t level,
TopkHeap &topk_heap) {
topk_heap.sort();
uint32_t max_neighbor_cnt = entity_.neighbor_cnt(level);
@@ -455,10 +459,10 @@ void HnswAlgorithm::update_neighbors(HnswDistCalculator &dc, node_id_t id,
return;
}
void HnswAlgorithm::reverse_update_neighbors(HnswDistCalculator &dc,
node_id_t id, level_t level,
node_id_t link_id, dist_t dist,
TopkHeap &update_heap) {
template <typename EntityType>
void HnswAlgorithm<EntityType>::reverse_update_neighbors(
HnswDistCalculator &dc, node_id_t id, level_t level, node_id_t link_id,
dist_t dist, TopkHeap &update_heap) {
const size_t max_neighbor_cnt = entity_.neighbor_cnt(level);
uint32_t lock_idx = id & kLockMask;
@@ -516,5 +520,10 @@ void HnswAlgorithm::reverse_update_neighbors(HnswDistCalculator &dc,
return;
}
// Explicit template instantiation
template class HnswAlgorithm<HnswMmapStreamerEntity>;
template class HnswAlgorithm<HnswBufferPoolStreamerEntity>;
template class HnswAlgorithm<HnswContiguousStreamerEntity>;
} // namespace core
} // namespace zvec
+34 -11
View File
@@ -14,41 +14,64 @@
#pragma once
#include <stdint.h>
#include <chrono>
#include <vector>
#include <ailego/internal/cpu_features.h>
#include <ailego/parallel/lock.h>
#include "hnsw_context.h"
#include "hnsw_dist_calculator.h"
#include "hnsw_entity.h"
#include "hnsw_streamer_entity.h"
namespace zvec {
namespace core {
//! hnsw graph algorithm implement
class HnswAlgorithm {
//! Non-template base class for HnswAlgorithm
class HnswAlgorithmBase {
public:
typedef std::unique_ptr<HnswAlgorithm> UPointer;
typedef std::unique_ptr<HnswAlgorithmBase> UPointer;
virtual ~HnswAlgorithmBase() = default;
virtual int cleanup() = 0;
virtual int add_node(node_id_t id, level_t level, HnswContext *ctx) = 0;
virtual int search(HnswContext *ctx) const = 0;
virtual int init() = 0;
virtual uint32_t get_random_level() const = 0;
};
//! hnsw graph algorithm implement, templated on EntityType
template <typename EntityType>
class HnswAlgorithm : public HnswAlgorithmBase {
public:
using MemBlockType = typename EntityType::MemoryBlock;
//! Constructor
explicit HnswAlgorithm(HnswEntity &entity);
explicit HnswAlgorithm(EntityType &entity)
: entity_(entity),
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
lock_pool_(kLockCnt) {}
//! Destructor
~HnswAlgorithm() = default;
~HnswAlgorithm() override = default;
//! Cleanup HnswAlgorithm
int cleanup();
int cleanup() override {
return 0;
}
//! Add a node to hnsw graph
//! @id: the node unique id
//! @level: a node will be add to graph in each level [0, level]
//! return 0 on success, or errCode in failure
int add_node(node_id_t id, level_t level, HnswContext *ctx);
int add_node(node_id_t id, level_t level, HnswContext *ctx) override;
//! do knn search in graph
//! return 0 on success, or errCode in failure. results saved in ctx
int search(HnswContext *ctx) const;
int search(HnswContext *ctx) const override;
//! Initiate HnswAlgorithm
int init() {
int init() override {
level_probas_.clear();
double level_mult =
1 / std::log(static_cast<double>(entity_.scaling_factor()));
@@ -67,7 +90,7 @@ class HnswAlgorithm {
//! Generate a random level
//! return graph level
uint32_t get_random_level() const {
uint32_t get_random_level() const override {
// gen rand float (0, 1)
double f = mt_() / static_cast<float>(mt_.max());
for (size_t level = 0; level < level_probas_.size(); level++) {
@@ -116,7 +139,7 @@ class HnswAlgorithm {
static constexpr uint32_t kLockCnt{1U << 8};
static constexpr uint32_t kLockMask{kLockCnt - 1U};
HnswEntity &entity_;
EntityType &entity_;
mutable std::mt19937 mt_{};
std::vector<double> level_probas_{};
+1
View File
@@ -42,6 +42,7 @@ class ChunkBroker {
CHUNK_TYPE_UPPER_NEIGHBOR = 4,
CHUNK_TYPE_NEIGHBOR_INDEX = 5,
CHUNK_TYPE_SPARSE_NODE = 6,
CHUNK_TYPE_NEIGHBOR_DIST = 7, // Vamana: per-node neighbor distances
CHUNK_TYPE_MAX = 8
};
static constexpr size_t kDefaultChunkSeqId = 0UL;
+136
View File
@@ -171,6 +171,142 @@ struct Neighbors {
IndexStorage::MemoryBlock neighbor_block;
};
//! Lightweight MemoryBlock for mmap mode: zero-cost construction/destruction
struct MmapMemoryBlock {
MmapMemoryBlock() = default;
explicit MmapMemoryBlock(void *data) : data_(data) {}
MmapMemoryBlock(const MmapMemoryBlock &) = default;
MmapMemoryBlock &operator=(const MmapMemoryBlock &) = default;
MmapMemoryBlock(MmapMemoryBlock &&) = default;
MmapMemoryBlock &operator=(MmapMemoryBlock &&) = default;
~MmapMemoryBlock() = default;
const void *data() const {
return data_;
}
void reset(void *data) {
data_ = data;
}
void *data_{nullptr};
};
//! Lightweight MemoryBlock for buffer pool mode: release on destruction
struct BufferPoolMemoryBlock {
BufferPoolMemoryBlock() = default;
BufferPoolMemoryBlock(ailego::VecBufferPoolHandle *handle, size_t block_id,
void *data)
: buffer_pool_handle_(handle), buffer_block_id_(block_id), data_(data) {}
BufferPoolMemoryBlock(const BufferPoolMemoryBlock &rhs)
: buffer_pool_handle_(rhs.buffer_pool_handle_),
buffer_block_id_(rhs.buffer_block_id_),
data_(rhs.data_) {
if (buffer_pool_handle_) {
buffer_pool_handle_->acquire_one(buffer_block_id_);
}
}
BufferPoolMemoryBlock &operator=(const BufferPoolMemoryBlock &rhs) {
if (this != &rhs) {
release();
buffer_pool_handle_ = rhs.buffer_pool_handle_;
buffer_block_id_ = rhs.buffer_block_id_;
data_ = rhs.data_;
if (buffer_pool_handle_) {
buffer_pool_handle_->acquire_one(buffer_block_id_);
}
}
return *this;
}
BufferPoolMemoryBlock(BufferPoolMemoryBlock &&rhs) noexcept
: buffer_pool_handle_(rhs.buffer_pool_handle_),
buffer_block_id_(rhs.buffer_block_id_),
data_(rhs.data_) {
rhs.buffer_pool_handle_ = nullptr;
rhs.data_ = nullptr;
}
BufferPoolMemoryBlock &operator=(BufferPoolMemoryBlock &&rhs) noexcept {
if (this != &rhs) {
release();
buffer_pool_handle_ = rhs.buffer_pool_handle_;
buffer_block_id_ = rhs.buffer_block_id_;
data_ = rhs.data_;
rhs.buffer_pool_handle_ = nullptr;
rhs.data_ = nullptr;
}
return *this;
}
~BufferPoolMemoryBlock() {
release();
}
const void *data() const {
return data_;
}
void reset(ailego::VecBufferPoolHandle *handle, size_t block_id, void *data) {
release();
buffer_pool_handle_ = handle;
buffer_block_id_ = block_id;
data_ = data;
}
private:
void release() {
if (buffer_pool_handle_) {
buffer_pool_handle_->release_one(buffer_block_id_);
buffer_pool_handle_ = nullptr;
}
data_ = nullptr;
}
ailego::VecBufferPoolHandle *buffer_pool_handle_{nullptr};
size_t buffer_block_id_{0};
void *data_{nullptr};
};
//! Typed Neighbors: holds a typed MemoryBlock to avoid runtime branching
template <typename MemBlockType>
struct NeighborsT {
NeighborsT() : cnt{0}, data{nullptr} {}
NeighborsT(uint32_t cnt_in, const node_id_t *data_in)
: cnt{cnt_in}, data{data_in} {}
explicit NeighborsT(const MemBlockType &mem_block)
: neighbor_block{mem_block} {
auto hd = reinterpret_cast<const NeighborsHeader *>(neighbor_block.data());
cnt = hd->neighbor_cnt;
data = hd->neighbors;
}
explicit NeighborsT(MemBlockType &&mem_block)
: neighbor_block{std::move(mem_block)} {
auto hd = reinterpret_cast<const NeighborsHeader *>(neighbor_block.data());
cnt = hd->neighbor_cnt;
data = hd->neighbors;
}
size_t size(void) const {
return cnt;
}
const node_id_t &operator[](size_t idx) const {
return data[idx];
}
uint32_t cnt;
const node_id_t *data;
MemBlockType neighbor_block;
};
//! level 0 neighbors offset
struct GraphNeighborMeta {
GraphNeighborMeta(size_t o, size_t cnt) : offset(o), neighbor_cnt(cnt) {}
+3
View File
@@ -108,5 +108,8 @@ static const std::string PARAM_HNSW_REDUCER_INDEX_NAME(
static const std::string PARAM_HNSW_REDUCER_EFCONSTRUCTION(
"proxima.hnsw.reducer.efconstruction");
static const std::string PARAM_HNSW_STREAMER_USE_CONTIGUOUS_MEMORY(
"proxima.hnsw.streamer.use_contiguous_memory");
} // namespace core
} // namespace zvec
+121 -72
View File
@@ -25,7 +25,7 @@
namespace zvec {
namespace core {
HnswStreamer::HnswStreamer() : entity_(stats_) {}
HnswStreamer::HnswStreamer() = default;
HnswStreamer::~HnswStreamer() {
if (state_ == STATE_INITED) {
@@ -68,7 +68,8 @@ int HnswStreamer::init(const IndexMeta &imeta, const ailego::Params &params) {
params.get(PARAM_HNSW_STREAMER_FORCE_PADDING_RESULT_ENABLE,
&force_padding_topk_enabled_);
params.get(PARAM_HNSW_STREAMER_USE_ID_MAP, &use_id_map_);
entity_.set_use_key_info_map(use_id_map_);
params.get(PARAM_HNSW_STREAMER_USE_CONTIGUOUS_MEMORY,
&use_contiguous_memory_);
params.get(PARAM_HNSW_STREAMER_DOCS_SOFT_LIMIT, &docs_soft_limit_);
if (docs_soft_limit_ > 0 && docs_soft_limit_ > docs_hard_limit_) {
@@ -143,6 +144,7 @@ int HnswStreamer::init(const IndexMeta &imeta, const ailego::Params &params) {
if (prune_cnt == 0UL) {
prune_cnt = upper_max_neighbor_cnt_;
}
prune_cnt_ = prune_cnt;
if (chunk_size_ == 0UL) {
chunk_size_ = HnswEntity::kDefaultChunkSize;
}
@@ -152,47 +154,21 @@ int HnswStreamer::init(const IndexMeta &imeta, const ailego::Params &params) {
return IndexError_InvalidArgument;
}
entity_.set_ef_construction(ef_construction_);
entity_.set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
entity_.set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
entity_.set_scaling_factor(scaling_factor_);
entity_.set_prune_cnt(prune_cnt);
entity_.set_vector_size(meta_.element_size());
entity_.set_chunk_size(chunk_size_);
entity_.set_filter_same_key(filter_same_key_);
entity_.set_get_vector(get_vector_enabled_);
entity_.set_min_neighbor_cnt(min_neighbor_cnt_);
int ret = entity_.init(docs_hard_limit_);
if (ret != 0) {
LOG_ERROR("Hnsw entity init failed for %s", IndexError::What(ret));
return ret;
}
LOG_DEBUG(
"Init params: maxIndexSize=%zu docsHardLimit=%zu docsSoftLimit=%zu "
"efConstruction=%u ef=%u upperMaxNeighborCnt=%u l0MaxNeighborCnt=%u "
"scalingFactor=%u maxScanRatio=%.3f minScanLimit=%zu maxScanLimit=%zu "
"bfEnabled=%d bruteFoceThreshold=%zu bfNegativeProbability=%.5f "
"checkCrcEnabled=%d pruneSize=%zu vectorSize=%u chunkSize=%zu "
"checkCrcEnabled=%d pruneSize=%u vectorSize=%u chunkSize=%zu "
"filterSameKey=%u getVectorEnabled=%u minNeighborCount=%u "
"forcePadding=%u ",
max_index_size_, docs_hard_limit_, docs_soft_limit_, ef_construction_,
ef_, upper_max_neighbor_cnt_, l0_max_neighbor_cnt_, scaling_factor_,
max_scan_ratio_, min_scan_limit_, max_scan_limit_, bf_enabled_,
bruteforce_threshold_, bf_negative_prob_, check_crc_enabled_, prune_cnt,
bruteforce_threshold_, bf_negative_prob_, check_crc_enabled_, prune_cnt_,
meta_.element_size(), chunk_size_, filter_same_key_, get_vector_enabled_,
min_neighbor_cnt_, force_padding_topk_enabled_);
alg_ = HnswAlgorithm::UPointer(new HnswAlgorithm(entity_));
ret = alg_->init();
if (ret != 0) {
return ret;
}
state_ = STATE_INITED;
return 0;
@@ -208,7 +184,9 @@ int HnswStreamer::cleanup(void) {
meta_.clear();
metric_.reset();
stats_.clear();
entity_.cleanup();
if (entity_) {
entity_->cleanup();
}
if (alg_) {
alg_->cleanup();
@@ -237,6 +215,26 @@ int HnswStreamer::cleanup(void) {
return 0;
}
int HnswStreamer::setup_entity() {
entity_->set_use_key_info_map(use_id_map_);
entity_->set_ef_construction(ef_construction_);
entity_->set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
entity_->set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
entity_->set_scaling_factor(scaling_factor_);
entity_->set_prune_cnt(prune_cnt_);
entity_->set_vector_size(meta_.element_size());
entity_->set_chunk_size(chunk_size_);
entity_->set_filter_same_key(filter_same_key_);
entity_->set_get_vector(get_vector_enabled_);
entity_->set_min_neighbor_cnt(min_neighbor_cnt_);
int ret = entity_->init(docs_hard_limit_);
if (ret != 0) {
LOG_ERROR("Hnsw entity init failed for %s", IndexError::What(ret));
}
return ret;
}
int HnswStreamer::open(IndexStorage::Pointer stg) {
LOG_INFO("HnswStreamer open");
@@ -244,15 +242,36 @@ int HnswStreamer::open(IndexStorage::Pointer stg) {
LOG_ERROR("Open storage failed, init streamer first!");
return IndexError_NoReady;
}
int ret = entity_.open(std::move(stg), max_index_size_, check_crc_enabled_);
// Create entity based on storage type
switch (stg->memory_block_type()) {
case IndexStorage::MemoryBlock::MBT_BUFFERPOOL: {
entity_ = std::make_unique<HnswBufferPoolStreamerEntity>(stats_);
break;
}
default: {
if (use_contiguous_memory_) {
entity_ = std::make_unique<HnswContiguousStreamerEntity>(stats_);
} else {
entity_ = std::make_unique<HnswMmapStreamerEntity>(stats_);
}
break;
}
}
int ret = setup_entity();
if (ret != 0) {
return ret;
}
ret = entity_->open(std::move(stg), max_index_size_, check_crc_enabled_);
if (ret != 0) {
return ret;
}
IndexMeta index_meta;
ret = entity_.get_index_meta(&index_meta);
ret = entity_->get_index_meta(&index_meta);
if (ret == IndexError_NoExist) {
// Set IndexMeta for the new index
ret = entity_.set_index_meta(meta_);
ret = entity_->set_index_meta(meta_);
if (ret != 0) {
LOG_ERROR("Failed to set index meta for %s", IndexError::What(ret));
return ret;
@@ -307,6 +326,36 @@ int HnswStreamer::open(IndexStorage::Pointer stg) {
search_batch_distance_ = metric_->query_metric()->batch_distance();
}
// Create algorithm based on entity storage mode
switch (entity_->storage_mode()) {
case HnswStorageMode::kBufferPool:
alg_ = HnswAlgorithmBase::UPointer(
new HnswAlgorithm<HnswBufferPoolStreamerEntity>(
static_cast<HnswBufferPoolStreamerEntity &>(*entity_)));
break;
case HnswStorageMode::kContiguous: {
auto &contiguous_entity =
static_cast<HnswContiguousStreamerEntity &>(*entity_);
int build_ret = contiguous_entity.build_contiguous_memory();
if (build_ret != 0) {
LOG_ERROR("Failed to build contiguous memory, ret=%d", build_ret);
return build_ret;
}
alg_ = HnswAlgorithmBase::UPointer(
new HnswAlgorithm<HnswContiguousStreamerEntity>(contiguous_entity));
break;
}
default:
alg_ =
HnswAlgorithmBase::UPointer(new HnswAlgorithm<HnswMmapStreamerEntity>(
static_cast<HnswMmapStreamerEntity &>(*entity_)));
break;
}
ret = alg_->init();
if (ret != 0) {
return ret;
}
state_ = STATE_OPENED;
magic_ = IndexContext::GenerateMagic();
@@ -318,8 +367,8 @@ int HnswStreamer::close(void) {
stats_.clear();
meta_.set_metric(metric_->name(), 0, metric_->params());
entity_.set_index_meta(meta_);
int ret = entity_.close();
entity_->set_index_meta(meta_);
int ret = entity_->close();
if (ret != 0) {
return ret;
}
@@ -332,8 +381,8 @@ int HnswStreamer::flush(uint64_t checkpoint) {
LOG_INFO("HnswStreamer flush checkpoint=%zu", (size_t)checkpoint);
meta_.set_metric(metric_->name(), 0, metric_->params());
entity_.set_index_meta(meta_);
return entity_.flush(checkpoint);
entity_->set_index_meta(meta_);
return entity_->flush(checkpoint);
}
int HnswStreamer::dump(const IndexDumper::Pointer &dumper) {
@@ -347,7 +396,7 @@ int HnswStreamer::dump(const IndexDumper::Pointer &dumper) {
LOG_ERROR("Failed to serialize meta into dumper.");
return ret;
}
return entity_.dump(dumper);
return entity_->dump(dumper);
}
IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
@@ -356,7 +405,7 @@ IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
return Context::Pointer();
}
HnswEntity::Pointer entity = entity_.clone();
HnswEntity::Pointer entity = entity_->clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("CreateContext clone init failed");
return Context::Pointer();
@@ -387,9 +436,9 @@ IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
if (meta_.streamer_params().get(PARAM_HNSW_STREAMER_ESTIMATE_DOC_COUNT,
&estimate_doc_count)) {
LOG_DEBUG("HnswStreamer doc_count[%zu] estimate[%zu]",
(size_t)entity_.doc_cnt(), (size_t)estimate_doc_count);
(size_t)entity_->doc_cnt(), (size_t)estimate_doc_count);
}
ctx->check_need_adjuct_ctx(std::max(entity_.doc_cnt(), estimate_doc_count));
ctx->check_need_adjuct_ctx(std::max(entity_->doc_cnt(), estimate_doc_count));
return Context::Pointer(ctx);
}
@@ -397,7 +446,7 @@ IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
IndexProvider::Pointer HnswStreamer::create_provider(void) const {
LOG_DEBUG("HnswStreamer create provider");
auto entity = entity_.clone();
auto entity = entity_->clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("Clone HnswEntity failed");
return nullptr;
@@ -407,7 +456,7 @@ IndexProvider::Pointer HnswStreamer::create_provider(void) const {
}
int HnswStreamer::update_context(HnswContext *ctx) const {
const HnswEntity::Pointer entity = entity_.clone();
const HnswEntity::Pointer entity = entity_->clone();
if (!entity) {
LOG_ERROR("Failed to clone search context entity");
return IndexError_Runtime;
@@ -442,15 +491,15 @@ int HnswStreamer::add_with_id_impl(uint32_t id, const void *query,
}
}
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
if (entity_.doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %u exceed [%s]", entity_.doc_cnt(),
if (ailego_unlikely(entity_->doc_cnt() >= docs_soft_limit_)) {
if (entity_->doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %u exceed [%s]", entity_->doc_cnt(),
PARAM_HNSW_STREAMER_DOCS_HARD_LIMIT.c_str());
const std::lock_guard<std::mutex> lk(mutex_);
(*stats_.mutable_discarded_count())++;
return IndexError_IndexFull;
} else {
LOG_WARN("Current docs %u exceed [%s]", entity_.doc_cnt(),
LOG_WARN("Current docs %u exceed [%s]", entity_->doc_cnt(),
PARAM_HNSW_STREAMER_DOCS_SOFT_LIMIT.c_str());
}
}
@@ -464,7 +513,7 @@ int HnswStreamer::add_with_id_impl(uint32_t id, const void *query,
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
if (metric_->support_train()) {
const std::lock_guard<std::mutex> lk(mutex_);
@@ -477,7 +526,7 @@ int HnswStreamer::add_with_id_impl(uint32_t id, const void *query,
}
level_t level = alg_->get_random_level();
ret = entity_.add_vector_with_id(level, id, query);
ret = entity_->add_vector_with_id(level, id, query);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer add vector failed");
(*stats_.mutable_discarded_count())++;
@@ -522,15 +571,15 @@ int HnswStreamer::add_impl(uint64_t pkey, const void *query,
}
}
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
if (entity_.doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %u exceed [%s]", entity_.doc_cnt(),
if (ailego_unlikely(entity_->doc_cnt() >= docs_soft_limit_)) {
if (entity_->doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %u exceed [%s]", entity_->doc_cnt(),
PARAM_HNSW_STREAMER_DOCS_HARD_LIMIT.c_str());
const std::lock_guard<std::mutex> lk(mutex_);
(*stats_.mutable_discarded_count())++;
return IndexError_IndexFull;
} else {
LOG_WARN("Current docs %u exceed [%s]", entity_.doc_cnt(),
LOG_WARN("Current docs %u exceed [%s]", entity_->doc_cnt(),
PARAM_HNSW_STREAMER_DOCS_SOFT_LIMIT.c_str());
}
}
@@ -544,7 +593,7 @@ int HnswStreamer::add_impl(uint64_t pkey, const void *query,
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
if (metric_->support_train()) {
const std::lock_guard<std::mutex> lk(mutex_);
@@ -558,7 +607,7 @@ int HnswStreamer::add_impl(uint64_t pkey, const void *query,
level_t level = alg_->get_random_level();
node_id_t id;
ret = entity_.add_vector(level, pkey, query, &id);
ret = entity_->add_vector(level, pkey, query, &id);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Hnsw streamer add vector failed");
(*stats_.mutable_discarded_count())++;
@@ -601,7 +650,7 @@ int HnswStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
return IndexError_Cast;
}
if (entity_.doc_cnt() <= ctx->get_bruteforce_threshold()) {
if (entity_->doc_cnt() <= ctx->get_bruteforce_threshold()) {
return search_bf_impl(query, qmeta, count, context);
}
@@ -616,7 +665,7 @@ int HnswStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
for (size_t q = 0; q < count; ++q) {
ctx->reset_query(query);
ret = alg_->search(ctx);
@@ -636,11 +685,11 @@ int HnswStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
}
void HnswStreamer::print_debug_info() {
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
if (entity_->get_key(id) == kInvalidKey) {
continue;
}
Neighbors neighbours = entity_.get_neighbors(0, id);
Neighbors neighbours = entity_->get_neighbors(0, id);
std::cout << "node: " << id << "; ";
if (neighbours.size() == 0) std::cout << std::endl;
for (uint32_t i = 0; i < neighbours.size(); ++i) {
@@ -654,7 +703,7 @@ void HnswStreamer::print_debug_info() {
}
}
// entity_.print_key_map();
// entity_->print_key_map();
}
int HnswStreamer::search_bf_impl(
@@ -694,19 +743,19 @@ int HnswStreamer::search_bf_impl(
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
return ctx->group_by()(entity_->get_key(id));
};
for (size_t q = 0; q < count; ++q) {
ctx->reset_query(query);
ctx->group_topk_heaps().clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
if (entity_->get_key(id) == kInvalidKey) {
continue;
}
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
if (!ctx->filter().is_valid() || !ctx->filter()(entity_->get_key(id))) {
dist_t dist = ctx->dist_calculator().batch_dist(id);
std::string group_id = group_by(id);
@@ -728,12 +777,12 @@ int HnswStreamer::search_bf_impl(
for (size_t q = 0; q < count; ++q) {
ctx->reset_query(query);
topk.clear();
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
if (entity_.get_key(id) == kInvalidKey) {
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
if (entity_->get_key(id) == kInvalidKey) {
continue;
}
if (!filter.is_valid() || !filter(entity_.get_key(id))) {
if (!filter.is_valid() || !filter(entity_->get_key(id))) {
dist_t dist = ctx->dist_calculator().batch_dist(id);
topk.emplace(id, dist);
}
@@ -788,7 +837,7 @@ int HnswStreamer::search_bf_by_p_keys_impl(
}
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
return ctx->group_by()(entity_.get_key(id));
return ctx->group_by()(entity_->get_key(id));
};
for (size_t q = 0; q < count; ++q) {
@@ -798,7 +847,7 @@ int HnswStreamer::search_bf_by_p_keys_impl(
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
uint64_t pk = p_keys[q][idx];
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
node_id_t id = entity_.get_id(pk);
node_id_t id = entity_->get_id(pk);
if (id != kInvalidNodeId) {
dist_t dist = ctx->dist_calculator().batch_dist(id);
std::string group_id = group_by(id);
@@ -824,7 +873,7 @@ int HnswStreamer::search_bf_by_p_keys_impl(
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
key_t pk = p_keys[q][idx];
if (!filter.is_valid() || !filter(pk)) {
node_id_t id = entity_.get_id(pk);
node_id_t id = entity_->get_id(pk);
if (id != kInvalidNodeId) {
dist_t dist = ctx->dist_calculator().batch_dist(id);
topk.emplace(id, dist);
+22 -6
View File
@@ -31,6 +31,17 @@ class HnswStreamer : public IndexStreamer {
HnswStreamer(const HnswStreamer &streamer) = delete;
HnswStreamer &operator=(const HnswStreamer &streamer) = delete;
public:
//! Retrieve the storage mode of the underlying entity. Returns
//! HnswStorageMode::kMmap when the entity has not been initialized yet.
//! Intended for introspection and debug/testing usage.
HnswStorageMode storage_mode() const {
if (!entity_) {
return HnswStorageMode::kMmap;
}
return entity_->storage_mode();
}
protected:
//! Initialize Streamer
virtual int init(const IndexMeta &imeta,
@@ -88,22 +99,22 @@ class HnswStreamer : public IndexStreamer {
//! Fetch vector by key
virtual const void *get_vector(uint64_t key) const override {
return entity_.get_vector_by_key(key);
return entity_->get_vector_by_key(key);
}
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_.get_vector_by_key(key, block);
return entity_->get_vector_by_key(key, block);
}
//! Fetch vector by id
virtual const void *get_vector_by_id(uint32_t id) const override {
return entity_.get_vector(id);
return entity_->get_vector(id);
}
virtual int get_vector_by_id(
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
return entity_.get_vector(id, block);
return entity_->get_vector(id, block);
}
//! Open index from file path
@@ -159,6 +170,9 @@ class HnswStreamer : public IndexStreamer {
}
private:
//! Configure and initialize the entity with saved parameters
int setup_entity();
//! To share ctx across streamer/searcher, we need to update the context for
//! current streamer/searcher
int update_context(HnswContext *ctx) const;
@@ -181,8 +195,8 @@ class HnswStreamer : public IndexStreamer {
}
};
HnswStreamerEntity entity_;
HnswAlgorithm::UPointer alg_;
std::unique_ptr<HnswStreamerEntity> entity_;
HnswAlgorithmBase::UPointer alg_;
IndexMeta meta_{};
IndexMetric::Pointer metric_{};
@@ -200,6 +214,7 @@ class HnswStreamer : public IndexStreamer {
size_t docs_hard_limit_{HnswEntity::kDefaultDocsHardLimit};
size_t docs_soft_limit_{0UL};
uint32_t min_neighbor_cnt_{0u};
uint32_t prune_cnt_{0u};
uint32_t upper_max_neighbor_cnt_{HnswEntity::kDefaultUpperMaxNeighborCnt};
uint32_t l0_max_neighbor_cnt_{HnswEntity::kDefaultL0MaxNeighborCnt};
uint32_t ef_{HnswEntity::kDefaultEf};
@@ -219,6 +234,7 @@ class HnswStreamer : public IndexStreamer {
bool get_vector_enabled_{false};
bool force_padding_topk_enabled_{false};
bool use_id_map_{true};
bool use_contiguous_memory_{false};
//! avoid add vector while dumping index
ailego::SharedMutex shared_mutex_{};
@@ -13,6 +13,9 @@
// limitations under the License.
#include "hnsw_streamer_entity.h"
#if defined(__linux__) || defined(__APPLE__)
#include <sys/mman.h>
#endif
#include <ailego/utility/memory_helper.h>
// #define DEBUG_PRINT
@@ -773,5 +776,214 @@ const HnswEntity::Pointer HnswStreamerEntity::clone() const {
return HnswEntity::Pointer(entity);
}
const HnswEntity::Pointer HnswMmapStreamerEntity::clone() const {
std::vector<Chunk::Pointer> node_chunks;
node_chunks.reserve(node_chunks_.size());
for (size_t i = 0UL; i < node_chunks_.size(); ++i) {
node_chunks.emplace_back(node_chunks_[i]->clone());
if (ailego_unlikely(!node_chunks[i])) {
LOG_ERROR("HnswMmapStreamerEntity get chunk failed in clone");
return HnswEntity::Pointer();
}
}
std::vector<Chunk::Pointer> upper_neighbor_chunks;
upper_neighbor_chunks.reserve(upper_neighbor_chunks_.size());
for (size_t i = 0UL; i < upper_neighbor_chunks_.size(); ++i) {
upper_neighbor_chunks.emplace_back(upper_neighbor_chunks_[i]->clone());
if (ailego_unlikely(!upper_neighbor_chunks[i])) {
LOG_ERROR("HnswMmapStreamerEntity get chunk failed in clone");
return HnswEntity::Pointer();
}
}
auto *entity = new (std::nothrow) HnswMmapStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_,
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_,
nullptr, nullptr);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswMmapStreamerEntity new failed");
}
return HnswEntity::Pointer(entity);
}
const HnswEntity::Pointer HnswContiguousStreamerEntity::clone() const {
std::vector<Chunk::Pointer> node_chunks;
node_chunks.reserve(node_chunks_.size());
for (size_t i = 0UL; i < node_chunks_.size(); ++i) {
node_chunks.emplace_back(node_chunks_[i]->clone());
if (ailego_unlikely(!node_chunks[i])) {
LOG_ERROR("HnswContiguousStreamerEntity get chunk failed in clone");
return HnswEntity::Pointer();
}
}
std::vector<Chunk::Pointer> upper_neighbor_chunks;
upper_neighbor_chunks.reserve(upper_neighbor_chunks_.size());
for (size_t i = 0UL; i < upper_neighbor_chunks_.size(); ++i) {
upper_neighbor_chunks.emplace_back(upper_neighbor_chunks_[i]->clone());
if (ailego_unlikely(!upper_neighbor_chunks[i])) {
LOG_ERROR("HnswContiguousStreamerEntity get chunk failed in clone");
return HnswEntity::Pointer();
}
}
auto *entity = new (std::nothrow) HnswContiguousStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_,
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_,
nullptr, nullptr);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswContiguousStreamerEntity new failed");
return HnswEntity::Pointer();
}
// Share contiguous memory with the clone (zero-copy)
entity->node_memory_ = node_memory_;
entity->node_base_ = node_base_;
entity->upper_neighbor_memory_ = upper_neighbor_memory_;
entity->upper_neighbor_base_ = upper_neighbor_base_;
entity->upper_chunk_offsets_ = upper_chunk_offsets_;
return HnswEntity::Pointer(entity);
}
// ============================================================================
// HnswContiguousStreamerEntity implementation
// ============================================================================
char *HnswContiguousStreamerEntity::allocate_contiguous(size_t size) {
if (size == 0) {
return nullptr;
}
#if defined(__linux__)
// Use mmap with MAP_ANONYMOUS for contiguous memory
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (ptr == MAP_FAILED) {
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
return nullptr;
}
// Request transparent huge pages
::madvise(ptr, size, MADV_HUGEPAGE);
return static_cast<char *>(ptr);
#elif defined(__APPLE__)
// macOS: use mmap with MAP_ANONYMOUS
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANON, -1, 0);
if (ptr == MAP_FAILED) {
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
return nullptr;
}
return static_cast<char *>(ptr);
#elif defined(_WIN32)
void *ptr = ::_aligned_malloc(size, ailego::MemoryHelper::PageSize());
if (!ptr) {
LOG_ERROR("_aligned_malloc failed for contiguous memory, size=%zu", size);
return nullptr;
}
return static_cast<char *>(ptr);
#else
// Fallback: aligned allocation
void *ptr = std::aligned_alloc(ailego::MemoryHelper::PageSize(), size);
if (!ptr) {
LOG_ERROR("aligned_alloc failed for contiguous memory, size=%zu", size);
return nullptr;
}
return static_cast<char *>(ptr);
#endif
}
int HnswContiguousStreamerEntity::build_contiguous_memory() {
node_memory_.reset();
node_base_ = nullptr;
upper_neighbor_memory_.reset();
upper_neighbor_base_ = nullptr;
upper_chunk_offsets_.clear();
const uint32_t total_docs = doc_cnt();
if (total_docs == 0) {
return 0;
}
// --- Build contiguous node memory ---
const size_t per_node = node_size();
const size_t total_node_data = static_cast<size_t>(total_docs) * per_node;
size_t node_memory_size = AlignHugePageSize(total_node_data);
char *raw_node = allocate_contiguous(node_memory_size);
if (!raw_node) {
return IndexError_Runtime;
}
node_memory_.reset(raw_node, ContiguousDeleter{node_memory_size});
node_base_ = raw_node;
// Copy node data from chunks into contiguous memory
// Each chunk holds node_cnt_per_chunk nodes, laid out at offset
// (id & mask) * node_size within the chunk.
const auto &chunks = node_chunks_;
const uint32_t nodes_per_chunk = 1U << node_index_mask_bits_;
for (size_t chunk_idx = 0; chunk_idx < chunks.size(); ++chunk_idx) {
const void *chunk_data = nullptr;
size_t data_size = chunks[chunk_idx]->data_size();
chunks[chunk_idx]->read(0, &chunk_data, data_size);
// Number of nodes in this chunk
uint32_t base_id = chunk_idx * nodes_per_chunk;
uint32_t count_in_chunk = std::min(nodes_per_chunk, total_docs - base_id);
// Copy each node's data
const char *src = static_cast<const char *>(chunk_data);
char *dst = node_base_ + static_cast<size_t>(base_id) * per_node;
std::memcpy(dst, src, static_cast<size_t>(count_in_chunk) * per_node);
}
// --- Build contiguous upper neighbor memory ---
const auto &upper_chunks = upper_neighbor_chunks_;
if (upper_chunks.empty()) {
return 0;
}
// Sync all upper neighbor chunks
sync_upper_neighbor_chunks(upper_chunks.size() - 1);
// Calculate cumulative offsets and total size
upper_chunk_offsets_.resize(upper_chunks.size());
size_t total_upper_size = 0;
for (size_t i = 0; i < upper_chunks.size(); ++i) {
upper_chunk_offsets_[i] = total_upper_size;
total_upper_size += upper_chunks[i]->data_size();
}
size_t upper_memory_size = AlignHugePageSize(total_upper_size);
char *raw_upper = allocate_contiguous(upper_memory_size);
if (!raw_upper) {
node_memory_.reset();
node_base_ = nullptr;
return IndexError_Runtime;
}
upper_neighbor_memory_.reset(raw_upper, ContiguousDeleter{upper_memory_size});
upper_neighbor_base_ = raw_upper;
// Copy upper neighbor data from chunks into contiguous memory
for (size_t i = 0; i < upper_chunks.size(); ++i) {
const void *chunk_data = nullptr;
size_t data_size = upper_chunks[i]->data_size();
upper_chunks[i]->read(0, &chunk_data, data_size);
std::memcpy(upper_neighbor_base_ + upper_chunk_offsets_[i], chunk_data,
data_size);
}
LOG_INFO(
"Built contiguous memory: node_size=%zu upper_neighbor_size=%zu "
"total_docs=%u node_chunks=%zu upper_chunks=%zu",
node_memory_size, upper_memory_size, total_docs, chunks.size(),
upper_chunks.size());
return 0;
}
} // namespace core
} // namespace zvec
+473 -4
View File
@@ -15,6 +15,11 @@
#pragma once
#include <iostream>
#include <memory>
#include <mutex>
#if defined(__linux__) || defined(__APPLE__)
#include <sys/mman.h>
#endif
#include <ailego/parallel/lock.h>
#include <sparsehash/dense_hash_map>
#include <sparsehash/dense_hash_set>
@@ -28,6 +33,10 @@
namespace zvec {
namespace core {
//! Storage mode for HnswStreamerEntity
enum class HnswStorageMode { kMmap = 0, kBufferPool = 1, kContiguous = 2 };
//! HnswStreamerEntity manage vector data, pkey, and node's neighbors
class HnswStreamerEntity : public HnswEntity {
public:
@@ -84,6 +93,11 @@ class HnswStreamerEntity : public HnswEntity {
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
//! Get the storage mode of this entity
virtual HnswStorageMode storage_mode() const {
return HnswStorageMode::kMmap;
}
void set_use_key_info_map(bool use_id_map) {
use_key_info_map_ = use_id_map;
LOG_DEBUG("use_key_info_map_: %d", (int)use_key_info_map_);
@@ -178,6 +192,22 @@ class HnswStreamerEntity : public HnswEntity {
std::cout << "key map ends" << std::endl;
}
//! Typed get_neighbors: returns NeighborsT<MemBlock> without runtime
//! branching on MemoryBlock type. MmapMemoryBlock specialization uses
//! pointer-based read; BufferPoolMemoryBlock uses MemoryBlock-based read.
template <typename MemBlock>
inline NeighborsT<MemBlock> get_neighbors_typed(level_t level,
node_id_t id) const;
//! Typed batch get_vector: fills vector<MemBlock> without runtime branching
template <typename MemBlock>
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MemBlock> &vec_blocks) const;
//! Typed get_key: reads key using typed MemBlock
template <typename MemBlock>
inline key_t get_key_typed(node_id_t id) const;
//! Get l0 neighbors size
inline size_t neighbors_size() const {
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
@@ -189,7 +219,7 @@ class HnswStreamerEntity : public HnswEntity {
}
private:
protected:
union UpperNeighborIndexMeta {
struct {
uint32_t level : 4;
@@ -200,6 +230,7 @@ class HnswStreamerEntity : public HnswEntity {
uint32_t data;
};
protected:
template <class Key, class T>
using HashMap = google::dense_hash_map<Key, T, std::hash<Key>>;
template <class Key, class T>
@@ -214,7 +245,7 @@ class HnswStreamerEntity : public HnswEntity {
using NIHashMap = HnswIndexHashMap<node_id_t, uint32_t>;
using NIHashMapPointer = std::shared_ptr<NIHashMap>;
//! Private construct, only be called by clone method
//! Clone construct, used by clone method in subclasses
HnswStreamerEntity(IndexStreamer::Stats &stats, const HNSWHeader &hd,
size_t chunk_size, uint32_t node_index_mask_bits,
uint32_t upper_neighbor_mask_bits, bool filter_same_key,
@@ -480,14 +511,23 @@ class HnswStreamerEntity : public HnswEntity {
return 0;
}
protected:
//! Expose sync_chunks for subclass use
inline void sync_node_chunks(size_t idx) const {
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, idx, &node_chunks_);
}
inline void sync_upper_neighbor_chunks(size_t idx) const {
sync_chunks(ChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR, idx,
&upper_neighbor_chunks_);
}
private:
HnswStreamerEntity(const HnswStreamerEntity &) = delete;
HnswStreamerEntity &operator=(const HnswStreamerEntity &) = delete;
static constexpr uint64_t kUpperHashMemoryInflateRatio = 2.0f;
private:
protected:
IndexStreamer::Stats &stats_;
HNSWHeader header_{};
std::mutex mutex_{};
size_t max_index_size_{0UL};
uint32_t chunk_size_{kDefaultChunkSize};
@@ -536,5 +576,434 @@ class HnswStreamerEntity : public HnswEntity {
ChunkBroker::Pointer broker_{}; // chunk broker
};
// --- Template specializations for typed MemoryBlock access ---
//! MmapMemoryBlock specialization: uses pointer-based Chunk::read
template <>
inline NeighborsT<MmapMemoryBlock>
HnswStreamerEntity::get_neighbors_typed<MmapMemoryBlock>(level_t level,
node_id_t id) const {
Chunk *chunk = nullptr;
size_t offset = 0UL;
size_t nbr_size = neighbor_size_;
if (level == 0UL) {
uint32_t chunk_idx = id >> node_index_mask_bits_;
offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
chunk = node_chunks_[chunk_idx].get();
} else {
auto p = get_upper_neighbor_chunk_loc(level, id);
chunk = upper_neighbor_chunks_[p.first].get();
offset = p.second;
nbr_size = upper_neighbor_size_;
}
ailego_assert_with(offset < chunk->data_size(), "invalid chunk offset");
const void *ptr = nullptr;
size_t ret = chunk->read(offset, &ptr, nbr_size);
if (ailego_unlikely(ret != nbr_size)) {
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
return NeighborsT<MmapMemoryBlock>();
}
MmapMemoryBlock block(const_cast<void *>(ptr));
return NeighborsT<MmapMemoryBlock>(std::move(block));
}
//! BufferPoolMemoryBlock specialization: uses MemoryBlock-based Chunk::read
template <>
inline NeighborsT<BufferPoolMemoryBlock>
HnswStreamerEntity::get_neighbors_typed<BufferPoolMemoryBlock>(
level_t level, node_id_t id) const {
Chunk *chunk = nullptr;
size_t offset = 0UL;
size_t nbr_size = neighbor_size_;
if (level == 0UL) {
uint32_t chunk_idx = id >> node_index_mask_bits_;
offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
chunk = node_chunks_[chunk_idx].get();
} else {
auto p = get_upper_neighbor_chunk_loc(level, id);
chunk = upper_neighbor_chunks_[p.first].get();
offset = p.second;
nbr_size = upper_neighbor_size_;
}
ailego_assert_with(offset < chunk->data_size(), "invalid chunk offset");
IndexStorage::MemoryBlock mem_block;
size_t ret = chunk->read(offset, mem_block, nbr_size);
if (ailego_unlikely(ret != nbr_size)) {
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
return NeighborsT<BufferPoolMemoryBlock>();
}
BufferPoolMemoryBlock block(mem_block.buffer_pool_handle_,
mem_block.buffer_block_id_, mem_block.data_);
mem_block.buffer_pool_handle_ = nullptr;
return NeighborsT<BufferPoolMemoryBlock>(std::move(block));
}
//! MmapMemoryBlock specialization for batch get_vector
template <>
inline int HnswStreamerEntity::get_vector_typed<MmapMemoryBlock>(
const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
auto loc = get_vector_chunk_loc(ids[i]);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
const void *ptr = nullptr;
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%u, read size=%zu, ret=%zu",
loc.second, read_size, ret);
return IndexError_ReadData;
}
vec_blocks[i].reset(const_cast<void *>(ptr));
}
return 0;
}
//! BufferPoolMemoryBlock specialization for batch get_vector
template <>
inline int HnswStreamerEntity::get_vector_typed<BufferPoolMemoryBlock>(
const node_id_t *ids, uint32_t count,
std::vector<BufferPoolMemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
auto loc = get_vector_chunk_loc(ids[i]);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
size_t read_size = vector_size();
IndexStorage::MemoryBlock mem_block;
size_t ret =
node_chunks_[loc.first]->read(loc.second, mem_block, read_size);
if (ailego_unlikely(ret != read_size)) {
LOG_ERROR("Read vector failed, offset=%u, read size=%zu, ret=%zu",
loc.second, read_size, ret);
return IndexError_ReadData;
}
vec_blocks[i] =
BufferPoolMemoryBlock(mem_block.buffer_pool_handle_,
mem_block.buffer_block_id_, mem_block.data_);
mem_block.buffer_pool_handle_ = nullptr;
}
return 0;
}
//! MmapMemoryBlock specialization for get_key
template <>
inline key_t HnswStreamerEntity::get_key_typed<MmapMemoryBlock>(
node_id_t id) const {
if (!use_key_info_map_) {
return id;
}
auto loc = get_key_chunk_loc(id);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
const void *ptr = nullptr;
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, sizeof(key_t));
if (ailego_unlikely(ret != sizeof(key_t))) {
LOG_ERROR("Read key failed, ret=%zu", ret);
return kInvalidKey;
}
return *reinterpret_cast<const key_t *>(ptr);
}
//! BufferPoolMemoryBlock specialization for get_key
template <>
inline key_t HnswStreamerEntity::get_key_typed<BufferPoolMemoryBlock>(
node_id_t id) const {
if (!use_key_info_map_) {
return id;
}
auto loc = get_key_chunk_loc(id);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
"invalid chunk offset");
IndexStorage::MemoryBlock key_block;
size_t ret =
node_chunks_[loc.first]->read(loc.second, key_block, sizeof(key_t));
if (ailego_unlikely(ret != sizeof(key_t))) {
LOG_ERROR("Read key failed, ret=%zu", ret);
return kInvalidKey;
}
return *reinterpret_cast<const key_t *>(key_block.data());
}
//! Typed entity subclass for mmap mode.
//! Caches chunk base addresses to eliminate virtual function calls on the
//! search hot path. For mmap mode, chunk data is memory-mapped at init time,
//! so we can directly compute pointers via base_addr + offset.
class HnswMmapStreamerEntity : public HnswStreamerEntity {
public:
using MemoryBlock = MmapMemoryBlock;
using TypedNeighbors = NeighborsT<MmapMemoryBlock>;
using HnswStreamerEntity::HnswStreamerEntity;
HnswStorageMode storage_mode() const override {
return HnswStorageMode::kMmap;
}
//! Override clone to return correct subclass type, so that
//! static_cast<const HnswMmapStreamerEntity&> in the algorithm is safe.
const HnswEntity::Pointer clone() const override;
inline TypedNeighbors get_neighbors_typed(level_t level, node_id_t id) const {
if (level == 0UL) {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
const char *base = get_node_chunk_base(chunk_idx);
MmapMemoryBlock block(const_cast<char *>(base + offset));
return TypedNeighbors(std::move(block));
}
// Upper level: use index to locate chunk and offset
auto it = upper_neighbor_index_->find(id);
ailego_assert_abort(it != upper_neighbor_index_->end(),
"Get upper neighbor header failed");
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
uint32_t chunk_idx = (meta->index) >> upper_neighbor_mask_bits_;
uint32_t offset = (((meta->index) & upper_neighbor_mask_) + level - 1) *
upper_neighbor_size_;
const char *base = get_upper_neighbor_chunk_base(chunk_idx);
MmapMemoryBlock block(const_cast<char *>(base + offset));
return TypedNeighbors(std::move(block));
}
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
uint32_t chunk_idx = ids[i] >> node_index_mask_bits_;
uint32_t offset = (ids[i] & node_index_mask_) * node_size();
const char *base = get_node_chunk_base(chunk_idx);
vec_blocks[i].reset(const_cast<char *>(base + offset));
}
return 0;
}
inline key_t get_key_typed(node_id_t id) const {
if (!use_key_info_map_) {
return id;
}
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
const char *base = get_node_chunk_base(chunk_idx);
return *reinterpret_cast<const key_t *>(base + offset);
}
private:
//! Get cached base address for a node chunk, syncing if needed
inline const char *get_node_chunk_base(uint32_t chunk_idx) const {
if (ailego_unlikely(chunk_idx >= node_chunk_bases_.size())) {
sync_node_chunk_bases(chunk_idx);
}
return node_chunk_bases_[chunk_idx];
}
//! Get cached base address for an upper neighbor chunk, syncing if needed
inline const char *get_upper_neighbor_chunk_base(uint32_t chunk_idx) const {
if (ailego_unlikely(chunk_idx >= upper_neighbor_chunk_bases_.size())) {
sync_upper_neighbor_chunk_bases(chunk_idx);
}
return upper_neighbor_chunk_bases_[chunk_idx];
}
//! Sync node chunk base addresses up to the given index
void sync_node_chunk_bases(uint32_t chunk_idx) const {
sync_node_chunks(chunk_idx);
const auto &chunks = node_chunks_;
for (size_t i = node_chunk_bases_.size(); i <= chunk_idx; ++i) {
const void *ptr = nullptr;
chunks[i]->read(0, &ptr, 1);
node_chunk_bases_.push_back(static_cast<const char *>(ptr));
}
}
//! Sync upper neighbor chunk base addresses up to the given index
void sync_upper_neighbor_chunk_bases(uint32_t chunk_idx) const {
sync_upper_neighbor_chunks(chunk_idx);
const auto &chunks = upper_neighbor_chunks_;
for (size_t i = upper_neighbor_chunk_bases_.size(); i <= chunk_idx; ++i) {
const void *ptr = nullptr;
chunks[i]->read(0, &ptr, 1);
upper_neighbor_chunk_bases_.push_back(static_cast<const char *>(ptr));
}
}
mutable std::vector<const char *> node_chunk_bases_{};
mutable std::vector<const char *> upper_neighbor_chunk_bases_{};
};
//! Typed entity subclass for buffer pool mode.
class HnswBufferPoolStreamerEntity : public HnswStreamerEntity {
public:
using MemoryBlock = BufferPoolMemoryBlock;
using TypedNeighbors = NeighborsT<BufferPoolMemoryBlock>;
using HnswStreamerEntity::HnswStreamerEntity;
HnswStorageMode storage_mode() const override {
return HnswStorageMode::kBufferPool;
}
inline TypedNeighbors get_neighbors_typed(level_t level, node_id_t id) const {
return HnswStreamerEntity::get_neighbors_typed<BufferPoolMemoryBlock>(level,
id);
}
inline int get_vector_typed(
const node_id_t *ids, uint32_t count,
std::vector<BufferPoolMemoryBlock> &vec_blocks) const {
return HnswStreamerEntity::get_vector_typed<BufferPoolMemoryBlock>(
ids, count, vec_blocks);
}
inline key_t get_key_typed(node_id_t id) const {
return HnswStreamerEntity::get_key_typed<BufferPoolMemoryBlock>(id);
}
};
//! Typed entity subclass for contiguous memory mode.
//! Allocates contiguous memory (with hugepage/THP support) and copies all
//! chunk data into it. Access is via a single base pointer + offset,
//! eliminating chunk-level indirection and maximizing memory locality.
class HnswContiguousStreamerEntity : public HnswMmapStreamerEntity {
public:
using HnswMmapStreamerEntity::HnswMmapStreamerEntity;
HnswStorageMode storage_mode() const override {
return HnswStorageMode::kContiguous;
}
//! Override clone to return correct subclass type.
//! Cloned entity shares contiguous memory via shared_ptr.
const HnswEntity::Pointer clone() const override;
~HnswContiguousStreamerEntity() = default;
//! Build contiguous memory from chunks after open.
//! Must be called after the entity is fully opened and all chunks are loaded.
int build_contiguous_memory();
//! Degrade to mmap mode by releasing contiguous memory and falling back
//! to chunk-based access.
void degrade_to_mmap() {
node_memory_.reset();
node_base_ = nullptr;
upper_neighbor_memory_.reset();
upper_neighbor_base_ = nullptr;
upper_chunk_offsets_.clear();
LOG_INFO("HNSW contiguous entity degraded to mmap mode for insertion");
}
bool is_contiguous() const {
return node_base_ != nullptr;
}
int add_vector(level_t level, key_t key, const void *vec,
node_id_t *id) override {
if (ailego_unlikely(is_contiguous())) degrade_to_mmap();
return HnswMmapStreamerEntity::add_vector(level, key, vec, id);
}
int add_vector_with_id(level_t level, node_id_t id,
const void *vec) override {
if (ailego_unlikely(is_contiguous())) degrade_to_mmap();
return HnswMmapStreamerEntity::add_vector_with_id(level, id, vec);
}
inline TypedNeighbors get_neighbors_typed(level_t level, node_id_t id) const {
if (ailego_likely(node_base_ != nullptr)) {
if (level == 0UL) {
const char *ptr = node_base_ + static_cast<size_t>(id) * node_size() +
vector_size() + sizeof(key_t);
MmapMemoryBlock block(const_cast<char *>(ptr));
return TypedNeighbors(std::move(block));
}
// Upper level: use index to locate global offset
auto it = upper_neighbor_index_->find(id);
ailego_assert_abort(it != upper_neighbor_index_->end(),
"Get upper neighbor header failed");
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
uint32_t chunk_idx = (meta->index) >> upper_neighbor_mask_bits_;
uint32_t local_idx = (meta->index) & upper_neighbor_mask_;
size_t global_offset =
upper_chunk_offsets_[chunk_idx] +
static_cast<size_t>(local_idx + level - 1) * upper_neighbor_size_;
const char *ptr = upper_neighbor_base_ + global_offset;
MmapMemoryBlock block(const_cast<char *>(ptr));
return TypedNeighbors(std::move(block));
}
return HnswMmapStreamerEntity::get_neighbors_typed(level, id);
}
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
if (ailego_likely(node_base_ != nullptr)) {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
const char *ptr =
node_base_ + static_cast<size_t>(ids[i]) * node_size();
vec_blocks[i].reset(const_cast<char *>(ptr));
}
return 0;
}
return HnswMmapStreamerEntity::get_vector_typed(ids, count, vec_blocks);
}
inline key_t get_key_typed(node_id_t id) const {
if (ailego_likely(node_base_ != nullptr)) {
if (!use_key_info_map_) {
return id;
}
const char *ptr =
node_base_ + static_cast<size_t>(id) * node_size() + vector_size();
return *reinterpret_cast<const key_t *>(ptr);
}
return HnswMmapStreamerEntity::get_key_typed(id);
}
protected:
//! Custom deleter for contiguous memory (munmap / _aligned_free / free)
//! Used by shared_ptr to properly release mmap'd memory.
struct ContiguousDeleter {
size_t size;
void operator()(char *ptr) const {
if (!ptr) return;
#if defined(__linux__) || defined(__APPLE__)
::munmap(ptr, size);
#elif defined(_WIN32)
::_aligned_free(ptr);
#else
std::free(ptr);
#endif
}
};
//! Shared ownership of contiguous memory (enables zero-copy clone)
std::shared_ptr<char> node_memory_{};
std::shared_ptr<char> upper_neighbor_memory_{};
//! Raw pointers for hot-path access (derived from shared_ptr)
char *node_base_{nullptr};
char *upper_neighbor_base_{nullptr};
//! Cumulative offsets for each upper neighbor chunk in contiguous memory
std::vector<size_t> upper_chunk_offsets_{};
private:
//! Allocate contiguous memory with hugepage/THP support
static char *allocate_contiguous(size_t size);
};
} // namespace core
} // namespace zvec
+11
View File
@@ -0,0 +1,11 @@
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
cc_library(
NAME core_knn_vamana
STATIC SHARED STRICT ALWAYS_LINK
SRCS *.cc
LIBS core_framework core_knn_hnsw sparsehash
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
VERSION "${PROXIMA_ZVEC_VERSION}"
)
@@ -0,0 +1,465 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "vamana_algorithm.h"
namespace zvec {
namespace core {
// ============================================================================
// add_node: Insert a new node into the Vamana graph.
//
// Algorithm (from DiskANN / Vamana paper):
// 1. GreedySearch from entry_point to find candidate neighbors
// 2. RobustPrune to select diverse neighbors for the new node
// 3. Update the new node's neighbor list
// 4. For each new neighbor, add reverse link; if over-degree, RobustPrune
// 5. If this is the first node, set it as entry point
// ============================================================================
template <typename EntityType>
int VamanaAlgorithm<EntityType>::add_node(node_id_t id, VamanaContext *ctx) {
// Lazily initialize distance storage on first insert
entity_.ensure_dist_storage();
spin_lock_.lock();
auto entry_point = entity_.entry_point();
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
entity_.update_entry_point(id);
spin_lock_.unlock();
return 0;
}
spin_lock_.unlock();
// Step 1: GreedySearch to find candidate neighbors
uint32_t search_list_size = entity_.search_list_size();
ctx->topk_heap().clear();
ctx->topk_heap().limit(search_list_size);
ctx->dist_calculator().clear_compare_cnt();
// Set query to the new node's vector
const void *query_vec = entity_.get_vector(id);
if (ailego_unlikely(query_vec == nullptr)) {
LOG_ERROR("Failed to get vector for node %u", id);
return IndexError_ReadData;
}
ctx->reset_query(query_vec);
greedy_search(entry_point, ctx);
// Step 2: RobustPrune to select diverse neighbors
auto &topk_heap = ctx->topk_heap();
robust_prune(id, topk_heap, entity_.alpha(), entity_.max_degree(), ctx);
// Copy result before reverse updates (which also call robust_prune)
auto pruned_neighbors = ctx->prune_result();
// Step 3: Update the new node's neighbor list and distances
entity_.update_neighbors(id, pruned_neighbors);
entity_.update_neighbor_dists(id, pruned_neighbors);
// Step 4: Reverse-link updates
update_neighbors_and_reverse_links(id, pruned_neighbors, ctx);
return 0;
}
// ============================================================================
// search: Greedy search for approximate nearest neighbors.
// ============================================================================
template <typename EntityType>
int VamanaAlgorithm<EntityType>::search(VamanaContext *ctx) const {
spin_lock_.lock();
auto entry_point = entity_.entry_point();
spin_lock_.unlock();
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
return 0;
}
auto &topk_heap = ctx->topk_heap();
topk_heap.clear();
// Use ef (query-time parameter) instead of entity.search_list_size()
// (build-time L parameter). search_list_size controls construction;
// ef controls search quality and is user-configurable at query time.
uint32_t ef_search = std::max(static_cast<uint32_t>(ctx->topk()), ctx->ef());
topk_heap.limit(ef_search);
greedy_search(entry_point, ctx);
return 0;
}
// ============================================================================
// greedy_search: Beam search from entry_point.
//
// Maintains a candidate min-heap (ordered by distance) and a visited set.
// At each step, pops the closest unvisited candidate, expands its neighbors,
// and adds unvisited neighbors to both the candidate heap and the topk heap.
// Stops when the closest candidate is farther than the worst in topk, or
// when the scan limit is reached.
// ============================================================================
template <typename EntityType>
void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
VamanaContext *ctx) const {
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
VamanaDistCalculator &dc = ctx->dist_calculator();
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
auto &topk_heap = ctx->topk_heap();
const IndexFilter &index_filter =
static_cast<const IndexContext *>(ctx)->filter();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (index_filter.is_valid()) {
filter = [&](node_id_t id) {
return index_filter(entity.get_key_typed(id));
};
}
candidates.clear();
visit.clear();
// Initialize with entry point using batch_dist (single-element batch).
// We must NOT use dc.dist(entry_point) here because dist() calls
// distance_() which is squared_euclidean_int8_distance (sign/abs trick,
// expects two raw int8 inputs), but query_ has been preprocessed by
// reset_query (+128 shift to uint8). batch_dist() correctly calls
// batch_distance_() which expects the preprocessed uint8 query.
dist_t entry_dist = dc.batch_dist(entry_point);
if (ailego_unlikely(dc.error())) {
return;
}
visit.set_visited(entry_point);
if (!filter(entry_point)) {
topk_heap.emplace(entry_point, entry_dist);
}
candidates.emplace(entry_point, entry_dist);
// Pre-allocate temporary vectors outside the hot loop to avoid
// per-iteration heap allocations. Sized to max_degree initially;
// resized inside the loop if actual neighbor count exceeds this.
uint32_t buf_capacity = entity.max_degree();
std::vector<node_id_t> neighbor_ids(buf_capacity);
std::vector<MemBlockType> neighbor_vec_blocks;
neighbor_vec_blocks.reserve(buf_capacity);
std::vector<float> dists(buf_capacity);
std::vector<const void *> neighbor_vecs(buf_capacity);
while (!candidates.empty() && !ctx->reach_scan_limit()) {
auto top = candidates.begin();
node_id_t current_node = top->first;
dist_t current_dist = top->second;
// Early termination: if the closest candidate is worse than the worst
// result in a full topk heap, we won't find anything better.
if (topk_heap.full() && current_dist > topk_heap[0].second) {
break;
}
candidates.pop();
// Expand neighbors using typed access (no virtual dispatch)
const auto neighbors = entity.get_neighbors_typed(current_node);
ailego_prefetch(neighbors.data);
// Resize buffers if this node has more neighbors than expected
if (neighbors.size() > buf_capacity) {
buf_capacity = neighbors.size();
neighbor_ids.resize(buf_capacity);
dists.resize(buf_capacity);
neighbor_vecs.resize(buf_capacity);
}
// Collect unvisited neighbors (reuse pre-allocated buffer)
uint32_t unvisited_count = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) continue;
visit.set_visited(node);
neighbor_ids[unvisited_count++] = node;
}
if (unvisited_count == 0) continue;
// Batch fetch vectors using typed access (reuse pre-allocated buffer)
neighbor_vec_blocks.clear();
int ret = entity.get_vector_typed(neighbor_ids.data(), unvisited_count,
neighbor_vec_blocks);
if (ailego_unlikely(ret != 0)) break;
// Prefetch for better cache performance
static constexpr uint32_t PREFETCH_BATCH = 2;
static constexpr uint32_t PREFETCH_STEP = 2;
for (uint32_t i = 0;
i < std::min(PREFETCH_BATCH * PREFETCH_STEP, unvisited_count); ++i) {
ailego_prefetch(neighbor_vec_blocks[i].data());
}
// Batch distance computation (reuse pre-allocated buffers)
for (uint32_t i = 0; i < unvisited_count; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
}
dc.batch_dist(neighbor_vecs.data(), unvisited_count, dists.data());
// Update candidates and topk.
// Unlike vanilla DiskANN which inserts all unvisited neighbors into
// the candidate queue unconditionally, we apply an early-pruning
// optimization: a neighbor is only inserted into the candidate queue
// (and topk_heap) if it could potentially improve the final results,
// i.e. either the topk heap is not yet full, or the neighbor is closer
// than the current worst result. This avoids expanding clearly
// unpromising branches and reduces the candidate queue size.
for (uint32_t i = 0; i < unvisited_count; ++i) {
node_id_t node = neighbor_ids[i];
dist_t node_dist = dists[i];
if ((!topk_heap.full()) || node_dist < topk_heap[0].second) {
candidates.emplace(node, node_dist);
if (!filter(node)) {
topk_heap.emplace(node, node_dist);
}
}
}
}
}
// ============================================================================
// robust_prune: Select up to max_degree diverse neighbors from candidates.
//
// Faithfully follows DiskANN's occlude_list algorithm:
// 1. Sort candidates by distance (ascending)
// 2. Truncate to max_occlusion_size (DiskANN's maxc parameter)
// 3. Multi-round alpha sweep: cur_alpha starts at 1.0, increments by *1.2
// each round until reaching alpha. This progressively relaxes the
// occlusion criterion.
// 4. For each candidate, compute occlude_factor as:
// max over all selected neighbors p of: dist(query, candidate) / dist(p,
// candidate)
// If occlude_factor > cur_alpha, the candidate is occluded in this round.
// 5. After all rounds, if _saturate_graph and alpha > 1, fill remaining
// slots with any un-selected candidates.
// ============================================================================
template <typename EntityType>
void VamanaAlgorithm<EntityType>::robust_prune(node_id_t id,
TopkHeap &candidates,
float alpha, uint32_t max_degree,
VamanaContext *ctx) const {
auto &result = ctx->prune_result();
result.clear();
if (candidates.size() == 0) return;
// Sort candidates by distance (ascending — closest first)
candidates.sort();
VamanaDistCalculator &dc = ctx->dist_calculator();
size_t n = candidates.size();
// Truncate to max_occlusion_size (DiskANN's maxc parameter)
size_t maxc = entity_.max_occlusion_size();
if (maxc > 0 && n > maxc) {
n = maxc;
}
// Pre-cache all candidate vectors at once
auto &vec_cache = ctx->prune_vec_cache();
vec_cache.resize(n);
for (size_t i = 0; i < n; ++i) {
vec_cache[i] = entity_.get_vector(candidates[i].first);
}
// occlude_factor: tracks the maximum occlusion ratio for each candidate
// (DiskANN: occlude_factor[t] = max over selected p of
// dist_to_query/dist_to_p)
auto &occlude_factor = ctx->prune_occlude_factor();
occlude_factor.assign(n, 0.0f);
// Pre-allocated buffers for batch distance computation
auto &batch_vecs = ctx->batch_vecs_buf();
auto &batch_dists = ctx->batch_dists_buf();
auto &batch_indices = ctx->batch_indices_buf();
batch_vecs.resize(n);
batch_dists.resize(n);
batch_indices.resize(n);
// Multi-round alpha sweep (DiskANN: cur_alpha starts at 1, increments *1.2)
float cur_alpha = 1.0f;
while (cur_alpha <= alpha + 1e-6f && result.size() < max_degree) {
for (size_t i = 0; i < n && result.size() < max_degree; ++i) {
if (occlude_factor[i] > cur_alpha) {
continue;
}
// Mark as consumed so it won't be reconsidered
occlude_factor[i] = std::numeric_limits<float>::max();
// Skip self-loops
if (candidates[i].first == id) continue;
const void *selected_vec = vec_cache[i];
if (ailego_unlikely(selected_vec == nullptr)) continue;
// Add this candidate as a neighbor
node_id_t candidate_id = candidates[i].first;
dist_t candidate_dist = candidates[i].second;
result.emplace_back(candidate_id, candidate_dist);
// Update occlude_factor for remaining candidates
// Collect candidates that haven't been consumed yet
uint32_t batch_count = 0;
for (size_t j = i + 1; j < n; ++j) {
if (occlude_factor[j] > alpha) continue; // already fully occluded
if (ailego_unlikely(vec_cache[j] == nullptr)) continue;
batch_vecs[batch_count] = vec_cache[j];
batch_indices[batch_count] = static_cast<uint32_t>(j);
batch_count++;
}
if (batch_count > 0) {
// Batch compute distances from selected candidate to remaining
dc.batch_dist_pair(selected_vec, batch_vecs.data(), batch_count,
batch_dists.data());
// DiskANN (L2/Cosine):
// occlude_factor[t] = max(occlude_factor[t], dist_to_query /
// dist_between)
// where dist_to_query = candidates[j].second (distance from query to j)
// dist_between = batch_dists[k] (distance from selected to j)
for (uint32_t k = 0; k < batch_count; ++k) {
uint32_t j = batch_indices[k];
float dist_selected_to_candidate = batch_dists[k];
if (dist_selected_to_candidate == 0.0f) {
occlude_factor[j] = std::numeric_limits<float>::max();
} else {
occlude_factor[j] =
std::max(occlude_factor[j],
candidates[j].second / dist_selected_to_candidate);
}
}
}
}
cur_alpha *= 1.2f;
}
// Saturate graph: if enabled and alpha > 1, fill remaining slots with any
// un-selected candidates. This improves graph connectivity (better recall)
// at the cost of slightly more distance computations during search.
// Configurable via proxima.vamana.streamer.saturate_graph (default: false,
// matching DiskANN's default behavior).
if (entity_.saturate_graph() && alpha > 1.0f) {
for (size_t i = 0; i < n && result.size() < max_degree; ++i) {
if (candidates[i].first == id) continue;
bool already_selected = false;
for (const auto &r : result) {
if (r.first == candidates[i].first) {
already_selected = true;
break;
}
}
if (!already_selected) {
result.emplace_back(candidates[i].first, candidates[i].second);
}
}
}
}
// ============================================================================
// update_neighbors_and_reverse_links: For each new neighbor of `id`,
// add a reverse link from neighbor back to `id`. If the neighbor's degree
// exceeds max_degree, prune it using RobustPrune.
// ============================================================================
template <typename EntityType>
void VamanaAlgorithm<EntityType>::update_neighbors_and_reverse_links(
node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &new_neighbors,
VamanaContext *ctx) {
for (const auto &[neighbor_id, dist] : new_neighbors) {
reverse_update_neighbor(id, neighbor_id, dist, ctx);
}
}
// ============================================================================
// reverse_update_neighbor: Add `id` as a neighbor of `neighbor_id`.
// If neighbor_id already has max_degree neighbors, collect all neighbors
// + the new one into a candidate set and RobustPrune.
// ============================================================================
template <typename EntityType>
void VamanaAlgorithm<EntityType>::reverse_update_neighbor(node_id_t id,
node_id_t neighbor_id,
dist_t dist,
VamanaContext *ctx) {
std::lock_guard<std::mutex> lock(lock_pool_[neighbor_id & kLockMask]);
const Neighbors current_neighbors = entity_.get_neighbors(neighbor_id);
uint32_t current_size = current_neighbors.size();
uint32_t max_deg = entity_.max_degree();
// Check if `id` is already a neighbor
for (uint32_t i = 0; i < current_size; ++i) {
if (current_neighbors[i] == id) return;
}
if (current_size < max_deg) {
// Simply append and record distance
entity_.add_neighbor(neighbor_id, current_size, id);
entity_.set_neighbor_dist(neighbor_id, current_size, dist);
return;
}
// Need to prune: collect current neighbors + new node into candidates
VamanaDistCalculator &dc = ctx->dist_calculator();
// Reuse update_heap from context instead of creating a new TopkHeap each time
TopkHeap &prune_candidates = ctx->update_heap();
prune_candidates.clear();
prune_candidates.limit(max_deg + 1);
// Add existing neighbors — use cached distances when available
const dist_t *cached_dists = entity_.get_neighbor_dists(neighbor_id);
if (cached_dists != nullptr) {
// Fast path: read distances from storage, no recomputation needed
for (uint32_t i = 0; i < current_size; ++i) {
prune_candidates.emplace(current_neighbors[i], cached_dists[i]);
}
} else {
// Fallback: compute distances (first time or dist storage not loaded)
const void *neighbor_vec = entity_.get_vector(neighbor_id);
if (ailego_unlikely(neighbor_vec == nullptr)) return;
for (uint32_t i = 0; i < current_size; ++i) {
node_id_t nbr = current_neighbors[i];
const void *nbr_vec = entity_.get_vector(nbr);
if (ailego_unlikely(nbr_vec == nullptr)) continue;
dist_t nbr_dist = dc.dist(neighbor_vec, nbr_vec);
prune_candidates.emplace(nbr, nbr_dist);
}
}
// Add the new reverse link
prune_candidates.emplace(id, dist);
// RobustPrune from neighbor_id's perspective
robust_prune(neighbor_id, prune_candidates, entity_.alpha(), max_deg, ctx);
// Update neighbor_id's neighbor list and distances
const auto &prune_result = ctx->prune_result();
entity_.update_neighbors(neighbor_id, prune_result);
entity_.update_neighbor_dists(neighbor_id, prune_result);
}
// Explicit template instantiation for all entity types
template class VamanaAlgorithm<VamanaMmapStreamerEntity>;
template class VamanaAlgorithm<VamanaBufferPoolStreamerEntity>;
template class VamanaAlgorithm<VamanaContiguousStreamerEntity>;
} // namespace core
} // namespace zvec
@@ -0,0 +1,115 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <chrono>
#include <mutex>
#include <vector>
#include <ailego/parallel/lock.h>
#include "vamana_context.h"
#include "vamana_dist_calculator.h"
#include "vamana_streamer_entity.h"
namespace zvec {
namespace core {
// Non-template base class providing a type-erased interface so that
// VamanaStreamer can hold a pointer without knowing the EntityType.
class VamanaAlgorithmBase {
public:
typedef std::unique_ptr<VamanaAlgorithmBase> UPointer;
virtual ~VamanaAlgorithmBase() = default;
virtual int cleanup() = 0;
// Insert a new node into the Vamana graph.
// The node's vector must already be stored in the entity.
virtual int add_node(node_id_t id, VamanaContext *ctx) = 0;
// Greedy search: find approximate nearest neighbors.
virtual int search(VamanaContext *ctx) const = 0;
virtual int init() = 0;
};
// Vamana graph algorithm, templated on EntityType for hot-path optimization.
// EntityType should be VamanaMmapStreamerEntity,
// VamanaBufferPoolStreamerEntity, or VamanaContiguousStreamerEntity.
//
// Core operations:
// - GreedySearch: beam search from entry point, expanding best candidates
// - RobustPrune: select diverse neighbors using alpha-based pruning
// - add_node: insert + prune + reverse-link update
template <typename EntityType>
class VamanaAlgorithm : public VamanaAlgorithmBase {
public:
using MemBlockType = typename EntityType::MemoryBlock;
explicit VamanaAlgorithm(EntityType &entity)
: entity_(entity), lock_pool_(kLockCnt) {}
~VamanaAlgorithm() override = default;
int cleanup() override {
return 0;
}
int init() override {
return 0;
}
// Insert node `id` into the graph. Its vector must already be in the entity.
int add_node(node_id_t id, VamanaContext *ctx) override;
// Greedy search from entry point. Results are stored in ctx->topk_heap().
int search(VamanaContext *ctx) const override;
private:
// GreedySearch: starting from entry_point, greedily expand the closest
// unvisited candidate until the search list is exhausted or scan limit
// is reached. Results accumulate in topk_heap.
void greedy_search(node_id_t entry_point, VamanaContext *ctx) const;
// RobustPrune: given a candidate set (topk_heap), select up to max_degree
// diverse neighbors using alpha-based distance comparison.
// Result is stored in ctx->prune_result().
void robust_prune(node_id_t id, TopkHeap &candidates, float alpha,
uint32_t max_degree, VamanaContext *ctx) const;
// Update node's neighbors and handle reverse links.
void update_neighbors_and_reverse_links(
node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &new_neighbors,
VamanaContext *ctx);
// Check if adding `id` as a reverse neighbor of `neighbor_id` requires
// pruning, and if so, prune neighbor_id's neighbor list.
void reverse_update_neighbor(node_id_t id, node_id_t neighbor_id, dist_t dist,
VamanaContext *ctx);
private:
VamanaAlgorithm(const VamanaAlgorithm &) = delete;
VamanaAlgorithm &operator=(const VamanaAlgorithm &) = delete;
static constexpr uint32_t kLockCnt{1U << 8};
static constexpr uint32_t kLockMask{kLockCnt - 1U};
EntityType &entity_;
mutable ailego::SpinMutex spin_lock_{};
std::vector<std::mutex> lock_pool_{};
};
} // namespace core
} // namespace zvec
+159
View File
@@ -0,0 +1,159 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "vamana_context.h"
#include <random>
#include "vamana_params.h"
namespace zvec {
namespace core {
VamanaContext::VamanaContext(size_t dimension,
const IndexMetric::Pointer &metric,
const VamanaEntity::Pointer &entity)
: IndexContext(metric),
entity_(entity),
dc_(entity.get(), metric, dimension),
metric_(metric) {}
VamanaContext::VamanaContext(const IndexMetric::Pointer &metric,
const VamanaEntity::Pointer &entity)
: IndexContext(metric),
entity_(entity),
dc_(entity.get(), metric),
metric_(metric) {}
VamanaContext::~VamanaContext() {
visit_filter_.destroy();
}
int VamanaContext::init(ContextType type) {
int ret;
uint32_t doc_cnt;
type_ = type;
results_.resize(1);
topk_heap_.limit(std::max(topk_, ef_));
update_heap_.limit(entity_->max_degree());
switch (type) {
case kBuilderContext:
ret = visit_filter_.init(VisitFilter::ByteMap, entity_->doc_cnt(),
max_scan_num_, filter_negative_prob_);
if (ret != 0) {
LOG_ERROR("Create visit filter failed, mode %d", filter_mode_);
return ret;
}
candidates_.limit(max_scan_num_);
break;
case kSearcherContext:
ret = visit_filter_.init(filter_mode_, entity_->doc_cnt(), max_scan_num_,
filter_negative_prob_);
if (ret != 0) {
LOG_ERROR("Create visit filter failed, mode %d", filter_mode_);
return ret;
}
candidates_.limit(max_scan_num_);
break;
case kStreamerContext:
doc_cnt = entity_->doc_cnt();
max_scan_num_ = compute_max_scan_num(doc_cnt);
reserve_max_doc_cnt_ = doc_cnt + compute_reserve_cnt(doc_cnt);
ret = visit_filter_.init(filter_mode_, reserve_max_doc_cnt_,
max_scan_num_, filter_negative_prob_);
if (ret != 0) {
LOG_ERROR("Create visit filter failed, mode %d", filter_mode_);
return ret;
}
candidates_.limit(max_scan_num_);
check_need_adjuct_ctx();
break;
default:
break;
}
return 0;
}
int VamanaContext::update_context(ContextType type, const IndexMeta &meta,
const IndexMetric::Pointer &metric,
const VamanaEntity::Pointer &entity,
uint32_t magic_num) {
if (magic_ == magic_num) {
return 0;
}
type_ = type;
entity_ = entity;
metric_ = metric;
magic_ = magic_num;
dc_.update(entity.get(), metric, meta.dimension());
return 0;
}
int VamanaContext::update(const ailego::Params &params) {
uint32_t ef = ef_;
params.get(PARAM_VAMANA_STREAMER_EF, &ef);
ef_ = ef;
topk_heap_.limit(std::max(topk_, ef_));
return 0;
}
void VamanaContext::topk_to_result(uint32_t idx) {
if (force_padding_topk_ && !topk_heap_.full() &&
topk_heap_.size() < entity_->doc_cnt()) {
this->fill_random_to_topk_full();
}
if (ailego_unlikely(topk_heap_.size() == 0)) {
return;
}
ailego_assert_with(idx < results_.size(), "invalid idx");
int size = std::min(topk_, static_cast<uint32_t>(topk_heap_.size()));
topk_heap_.sort();
results_[idx].clear();
for (int i = 0; i < size; ++i) {
auto score = topk_heap_[i].second;
if (score > this->threshold()) {
break;
}
node_id_t id = topk_heap_[i].first;
if (fetch_vector_) {
results_[idx].emplace_back(entity_->get_key(id), score, id,
entity_->get_vector(id));
} else {
results_[idx].emplace_back(entity_->get_key(id), score, id);
}
}
}
void VamanaContext::fill_random_to_topk_full() {
std::mt19937 rng(42);
uint32_t doc_cnt = entity_->doc_cnt();
uint32_t max_attempts = doc_cnt * 2;
uint32_t attempts = 0;
while (!topk_heap_.full() && doc_cnt > 0 && attempts < max_attempts) {
node_id_t random_id = rng() % doc_cnt;
if (entity_->get_key(random_id) != kInvalidKey) {
dist_t random_dist = dc_.dist(random_id);
topk_heap_.emplace_back(random_id, random_dist);
}
++attempts;
}
}
} // namespace core
} // namespace zvec
+315
View File
@@ -0,0 +1,315 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/core/framework/index_context.h>
#include "utility/visit_filter.h"
#include "vamana_dist_calculator.h"
#include "vamana_entity.h"
namespace zvec {
namespace core {
class VamanaContext : public IndexContext {
public:
typedef std::unique_ptr<VamanaContext> Pointer;
enum ContextType {
kUnknownContext = 0,
kSearcherContext = 1,
kBuilderContext = 2,
kStreamerContext = 3
};
VamanaContext(size_t dimension, const IndexMetric::Pointer &metric,
const VamanaEntity::Pointer &entity);
VamanaContext(const IndexMetric::Pointer &metric,
const VamanaEntity::Pointer &entity);
virtual ~VamanaContext();
virtual void set_topk(uint32_t val) override {
topk_ = val;
topk_heap_.limit(std::max(val, ef_));
}
virtual const IndexDocumentList &result(void) const override {
return results_[0];
}
virtual const IndexDocumentList &result(size_t idx) const override {
return results_[idx];
}
virtual IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
virtual uint32_t magic(void) const override {
return magic_;
}
virtual void set_debug_mode(bool enable) override {
debug_mode_ = enable;
}
virtual bool debug_mode(void) const override {
return debug_mode_;
}
virtual std::string debug_string(void) const override {
char buf[4096];
size_t size = snprintf(buf, sizeof(buf), "scan_cnt=%zu", get_scan_num());
return std::string(buf, size);
}
virtual int update(const ailego::Params &params) override;
int init(ContextType type);
int update_context(ContextType type, const IndexMeta &meta,
const IndexMetric::Pointer &metric,
const VamanaEntity::Pointer &entity, uint32_t magic_num);
inline const VamanaEntity &get_entity() const {
return *entity_;
}
inline void resize_results(size_t size) {
results_.resize(size);
}
inline void topk_to_result() {
topk_to_result(0);
}
void topk_to_result(uint32_t idx);
inline void reset_query(const void *query) {
if (auto query_preprocess_func = index_metric_->get_query_preprocess_func();
query_preprocess_func != nullptr) {
size_t dim = dc_.dimension();
preprocess_buffer_.resize(dim);
memcpy(preprocess_buffer_.data(), query, dim);
query_preprocess_func(preprocess_buffer_.data(), dim);
query = preprocess_buffer_.data();
}
dc_.reset_query(query);
dc_.clear_compare_cnt();
}
inline VamanaDistCalculator &dist_calculator() {
return dc_;
}
inline TopkHeap &topk_heap() {
return topk_heap_;
}
inline TopkHeap &update_heap() {
return update_heap_;
}
inline VisitFilter &visit_filter() {
return visit_filter_;
}
inline CandidateHeap &candidates() {
return candidates_;
}
// Pre-allocated buffers for robust_prune optimization
inline std::vector<const void *> &prune_vec_cache() {
return prune_vec_cache_;
}
inline std::vector<uint8_t> &prune_active() {
return prune_active_;
}
inline std::vector<float> &prune_occlude_factor() {
return prune_occlude_factor_;
}
inline std::vector<std::pair<node_id_t, dist_t>> &prune_result() {
return prune_result_;
}
inline std::vector<const void *> &batch_vecs_buf() {
return batch_vecs_buf_;
}
inline std::vector<float> &batch_dists_buf() {
return batch_dists_buf_;
}
inline std::vector<uint32_t> &batch_indices_buf() {
return batch_indices_buf_;
}
inline void set_max_scan_num(uint32_t max_scan_num) {
max_scan_num_ = max_scan_num;
}
inline void set_ef(uint32_t v) {
ef_ = v;
}
inline uint32_t ef() const {
return ef_;
}
inline void set_max_scan_ratio(float v) {
max_scan_ratio_ = v;
}
virtual void set_magic(uint32_t v) {
magic_ = v;
}
virtual void set_force_padding_topk(bool v) {
force_padding_topk_ = v;
}
void set_bruteforce_threshold(uint32_t v) override {
bruteforce_threshold_ = v;
}
inline uint32_t get_bruteforce_threshold() const {
return bruteforce_threshold_;
}
void set_fetch_vector(bool v) override {
fetch_vector_ = v;
}
bool fetch_vector() const override {
return fetch_vector_;
}
void set_max_scan_limit(size_t v) {
max_scan_limit_ = v;
}
void set_min_scan_limit(size_t v) {
min_scan_limit_ = v;
}
void set_filter_mode(VisitFilter::Mode mode) {
filter_mode_ = mode;
}
void set_filter_negative_probability(float prob) {
filter_negative_prob_ = prob;
}
void reset(void) override {
dc_.clear();
for (auto &it : results_) {
it.clear();
}
IndexContext::reset_filter();
IndexContext::reset_threshold();
IndexContext::set_fetch_vector(false);
}
inline void check_need_adjuct_ctx(void) {
check_need_adjuct_ctx(entity_->doc_cnt());
}
inline void check_need_adjuct_ctx(uint32_t doc_cnt) {
if (ailego_unlikely(doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_)) {
while (doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_) {
reserve_max_doc_cnt_ =
reserve_max_doc_cnt_ + compute_reserve_cnt(reserve_max_doc_cnt_);
}
uint32_t max_scan_cnt = compute_max_scan_num(reserve_max_doc_cnt_);
max_scan_num_ = max_scan_cnt;
visit_filter_.reset(reserve_max_doc_cnt_, max_scan_cnt);
candidates_.clear();
candidates_.limit(max_scan_num_);
}
}
inline size_t get_scan_num() const {
return dc_.compare_cnt();
}
inline uint64_t reach_scan_limit() const {
return dc_.compare_cnt() >= max_scan_num_;
}
inline bool error() const {
return dc_.error();
}
inline void clear() {
dc_.clear();
for (auto &it : results_) {
it.clear();
}
}
inline uint32_t topk() const override {
return topk_;
}
inline void update_dist_caculator_distance(
const IndexMetric::MatrixDistance &distance,
const IndexMetric::MatrixBatchDistance &batch_distance) {
dc_.update_distance(distance, batch_distance);
}
private:
void fill_random_to_topk_full(void);
inline size_t compute_reserve_cnt(uint32_t cur_doc) const {
if (cur_doc > kMaxReserveDocCnt) return kMaxReserveDocCnt;
if (cur_doc < kMinReserveDocCnt) return kMinReserveDocCnt;
return cur_doc;
}
inline uint32_t compute_max_scan_num(uint32_t max_doc_cnt) const {
uint32_t max_scan = max_doc_cnt * max_scan_ratio_;
if (max_scan < min_scan_limit_) max_scan = min_scan_limit_;
if (max_scan > max_scan_limit_) max_scan = max_scan_limit_;
return max_scan;
}
constexpr static uint32_t kTriggerReserveCnt = 4096UL;
constexpr static uint32_t kMinReserveDocCnt = 4096UL;
constexpr static uint32_t kMaxReserveDocCnt = 128 * 1024UL;
VamanaEntity::Pointer entity_;
VamanaDistCalculator dc_;
IndexMetric::Pointer metric_;
bool debug_mode_{false};
bool force_padding_topk_{false};
uint32_t max_scan_num_{0};
uint32_t reserve_max_doc_cnt_{kMinReserveDocCnt};
uint32_t topk_{0};
uint32_t ef_{VamanaEntity::kDefaultEf};
float max_scan_ratio_{VamanaEntity::kDefaultScanRatio};
size_t max_scan_limit_{VamanaEntity::kDefaultMaxScanLimit};
size_t min_scan_limit_{VamanaEntity::kDefaultMinScanLimit};
uint32_t magic_{0U};
std::vector<IndexDocumentList> results_{};
TopkHeap topk_heap_{};
TopkHeap update_heap_{};
CandidateHeap candidates_{};
VisitFilter visit_filter_{};
uint32_t bruteforce_threshold_{};
bool fetch_vector_{false};
uint32_t type_{kUnknownContext};
std::string preprocess_buffer_;
// Pre-allocated buffers for robust_prune optimization
std::vector<const void *> prune_vec_cache_;
std::vector<uint8_t> prune_active_;
std::vector<float> prune_occlude_factor_;
std::vector<std::pair<node_id_t, dist_t>> prune_result_;
std::vector<const void *> batch_vecs_buf_;
std::vector<float> batch_dists_buf_;
std::vector<uint32_t> batch_indices_buf_;
VisitFilter::Mode filter_mode_{VisitFilter::ByteMap};
float filter_negative_prob_{VamanaEntity::kDefaultBFNegativeProbability};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,197 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/core/framework/index_meta.h>
#include "vamana_entity.h"
namespace zvec {
namespace core {
class VamanaDistCalculator {
public:
typedef std::shared_ptr<VamanaDistCalculator> Pointer;
VamanaDistCalculator(const VamanaEntity *entity,
const IndexMetric::Pointer &metric, uint32_t dim)
: entity_(entity),
distance_(metric->distance()),
batch_distance_(metric->batch_distance()),
query_(nullptr),
dim_(dim),
compare_cnt_(0) {}
VamanaDistCalculator(const VamanaEntity *entity,
const IndexMetric::Pointer &metric, uint32_t dim,
const void *query)
: entity_(entity),
distance_(metric->distance()),
batch_distance_(metric->batch_distance()),
query_(query),
dim_(dim),
compare_cnt_(0) {}
VamanaDistCalculator(const VamanaEntity *entity,
const IndexMetric::Pointer &metric)
: entity_(entity),
distance_(metric->distance()),
batch_distance_(metric->batch_distance()),
query_(nullptr),
dim_(0),
compare_cnt_(0) {}
void update(const VamanaEntity *entity, const IndexMetric::Pointer &metric) {
entity_ = entity;
distance_ = metric->distance();
batch_distance_ = metric->batch_distance();
}
void update(const VamanaEntity *entity, const IndexMetric::Pointer &metric,
uint32_t dim) {
entity_ = entity;
distance_ = metric->distance();
batch_distance_ = metric->batch_distance();
dim_ = dim;
}
inline void update_distance(
const IndexMetric::MatrixDistance &distance,
const IndexMetric::MatrixBatchDistance &batch_distance) {
distance_ = distance;
batch_distance_ = batch_distance;
}
inline void reset_query(const void *query) {
error_ = false;
query_ = query;
}
inline dist_t dist(const void *vec_lhs, const void *vec_rhs) {
if (ailego_unlikely(vec_lhs == nullptr || vec_rhs == nullptr)) {
LOG_ERROR("Nullptr of dense vector");
error_ = true;
return 0.0f;
}
float score{0.0f};
distance_(vec_lhs, vec_rhs, dim_, &score);
return score;
}
inline dist_t dist(const void *vec) {
compare_cnt_++;
return dist(vec, query_);
}
inline dist_t dist(node_id_t id) {
compare_cnt_++;
const void *feat = entity_->get_vector(id);
if (ailego_unlikely(feat == nullptr)) {
LOG_ERROR("Get nullptr vector, id=%u", id);
error_ = true;
return 0.0f;
}
return dist(feat, query_);
}
inline dist_t dist(node_id_t lhs, node_id_t rhs) {
compare_cnt_++;
const void *feat = entity_->get_vector(lhs);
const void *query = entity_->get_vector(rhs);
if (ailego_unlikely(feat == nullptr || query == nullptr)) {
LOG_ERROR("Get nullptr vector");
error_ = true;
return 0.0f;
}
return dist(feat, query);
}
inline void batch_dist(const void **vecs, uint32_t count, float *dists) {
compare_cnt_ += count;
batch_distance_(vecs, query_, count, dim_, dists);
}
// Single-node batch distance: compute distance between query and a stored
// node using batch_distance_. Consistent with HnswDistCalculator::batch_dist.
inline dist_t batch_dist(node_id_t id) {
compare_cnt_++;
const void *feat = entity_->get_vector(id);
if (ailego_unlikely(feat == nullptr)) {
LOG_ERROR("Get nullptr vector, id=%u", id);
error_ = true;
return 0.0f;
}
dist_t score = 0;
batch_distance_(&feat, query_, 1, dim_, &score);
return score;
}
// Batch distance computation between a base vector and multiple target
// vectors. Does NOT use query_ and does NOT increment compare_cnt. Used for
// inter-candidate distance computation in robust_prune.
//
// Uses the single distance function (distance_) in a loop rather than
// batch_distance_, because batch_distance_ (turbo AVX512-VNNI) expects
// the second argument to be a preprocessed uint8 query (+128 shift),
// while base_vec here is a raw int8 stored vector. The single distance
// function (AVX2 sign/abs trick) correctly handles two raw int8 inputs.
inline void batch_dist_pair(const void *base_vec, const void **vecs,
uint32_t count, float *dists) {
for (uint32_t i = 0; i < count; ++i) {
distance_(base_vec, vecs[i], dim_, &dists[i]);
}
}
dist_t operator()(const void *vec) {
return dist(vec);
}
dist_t operator()(node_id_t i) {
return dist(i);
}
dist_t operator()(node_id_t lhs, node_id_t rhs) {
return dist(lhs, rhs);
}
inline void clear() {
compare_cnt_ = 0;
error_ = false;
}
inline void clear_compare_cnt() {
compare_cnt_ = 0;
}
inline bool error() const {
return error_;
}
inline uint32_t compare_cnt() const {
return compare_cnt_;
}
inline uint32_t dimension() const {
return dim_;
}
private:
VamanaDistCalculator(const VamanaDistCalculator &) = delete;
VamanaDistCalculator &operator=(const VamanaDistCalculator &) = delete;
const VamanaEntity *entity_;
IndexMetric::MatrixDistance distance_;
IndexMetric::MatrixBatchDistance batch_distance_;
const void *query_;
uint32_t dim_;
uint32_t compare_cnt_;
bool error_{false};
};
} // namespace core
} // namespace zvec
+203
View File
@@ -0,0 +1,203 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "vamana_entity.h"
#include <zvec/ailego/hash/crc32c.h>
namespace zvec {
namespace core {
const std::string VamanaEntity::kGraphHeaderSegmentId = "vamana.graph.header";
const std::string VamanaEntity::kGraphFeaturesSegmentId =
"vamana.graph.features";
const std::string VamanaEntity::kGraphKeysSegmentId = "vamana.graph.keys";
const std::string VamanaEntity::kGraphNeighborsSegmentId =
"vamana.graph.neighbors";
const std::string VamanaEntity::kGraphOffsetsSegmentId = "vamana.graph.offsets";
const std::string VamanaEntity::kGraphMappingSegmentId = "vamana.graph.mapping";
const std::string VamanaEntity::kGraphNeighborDistsSegmentId =
"vamana.graph.neighbor_dists";
int VamanaEntity::CalcAndAddPadding(const IndexDumper::Pointer &dumper,
size_t data_size, size_t *padding_size) {
*padding_size = AlignSize(data_size) - data_size;
if (*padding_size == 0) {
return 0;
}
std::string padding(*padding_size, '\0');
if (dumper->write(padding.data(), *padding_size) != *padding_size) {
LOG_ERROR("Append padding failed, size %lu", *padding_size);
return IndexError_WriteData;
}
return 0;
}
int64_t VamanaEntity::dump_segment(const IndexDumper::Pointer &dumper,
const std::string &segment_id,
const void *data, size_t size) const {
size_t len = dumper->write(data, size);
if (len != size) {
LOG_ERROR("Dump segment %s data failed, expect: %lu, actual: %lu",
segment_id.c_str(), size, len);
return IndexError_WriteData;
}
size_t padding_size = AlignSize(size) - size;
if (padding_size > 0) {
std::string padding(padding_size, '\0');
if (dumper->write(padding.data(), padding_size) != padding_size) {
LOG_ERROR("Append padding failed, size %lu", padding_size);
return IndexError_WriteData;
}
}
uint32_t crc = ailego::Crc32c::Hash(data, size);
int ret = dumper->append(segment_id, size, padding_size, crc);
if (ret != 0) {
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
return ret;
}
return len + padding_size;
}
int64_t VamanaEntity::dump_header(const IndexDumper::Pointer &dumper,
const VamanaHeader &hd) const {
return dump_segment(dumper, kGraphHeaderSegmentId, &hd.graph, hd.graph.size);
}
int64_t VamanaEntity::dump_vectors(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping) const {
size_t total_size = doc_cnt() * vector_size();
std::vector<uint8_t> buffer(total_size);
for (node_id_t i = 0; i < doc_cnt(); ++i) {
node_id_t old_id = reorder_mapping[i];
const void *vec = get_vector(old_id);
if (vec == nullptr) {
LOG_ERROR("Get vector failed for node %u", old_id);
return IndexError_ReadData;
}
memcpy(buffer.data() + static_cast<size_t>(i) * vector_size(), vec,
vector_size());
}
return dump_segment(dumper, kGraphFeaturesSegmentId, buffer.data(),
total_size);
}
int64_t VamanaEntity::dump_neighbors(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const {
size_t nbr_size = neighbors_size();
size_t total_size = doc_cnt() * nbr_size;
std::vector<uint8_t> buffer(total_size, 0);
for (node_id_t i = 0; i < doc_cnt(); ++i) {
node_id_t old_id = reorder_mapping[i];
const Neighbors nbrs = get_neighbors(old_id);
auto *hd = reinterpret_cast<NeighborsHeader *>(
buffer.data() + static_cast<size_t>(i) * nbr_size);
hd->neighbor_cnt = nbrs.size();
for (uint32_t j = 0; j < nbrs.size(); ++j) {
hd->neighbors[j] = neighbor_mapping[nbrs[j]];
}
}
return dump_segment(dumper, kGraphNeighborsSegmentId, buffer.data(),
total_size);
}
int64_t VamanaEntity::dump_neighbor_dists(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping) const {
if (!dist_storage_loaded()) {
// No distance data to dump — this is fine for read-only indices
return 0;
}
uint32_t max_deg = static_cast<uint32_t>(max_degree());
size_t dist_entry = max_deg * sizeof(dist_t);
size_t total_size = doc_cnt() * dist_entry;
std::vector<uint8_t> buffer(total_size, 0);
for (node_id_t i = 0; i < doc_cnt(); ++i) {
node_id_t old_id = reorder_mapping[i];
const dist_t *dists = get_neighbor_dists(old_id);
if (dists != nullptr) {
memcpy(buffer.data() + static_cast<size_t>(i) * dist_entry, dists,
dist_entry);
}
}
return dump_segment(dumper, kGraphNeighborDistsSegmentId, buffer.data(),
total_size);
}
int64_t VamanaEntity::dump_mapping_segment(const IndexDumper::Pointer &dumper,
const key_t *keys) const {
size_t total_size = doc_cnt() * sizeof(key_t);
return dump_segment(dumper, kGraphMappingSegmentId, keys, total_size);
}
void VamanaEntity::reshuffle_vectors(std::vector<node_id_t> *n2o_mapping,
std::vector<node_id_t> *o2n_mapping,
key_t *keys) const {
uint32_t count = doc_cnt();
n2o_mapping->resize(count);
o2n_mapping->resize(count);
// Simple identity mapping for now (can be optimized with BFS traversal)
for (uint32_t i = 0; i < count; ++i) {
(*n2o_mapping)[i] = i;
(*o2n_mapping)[i] = i;
keys[i] = get_key(i);
}
}
int64_t VamanaEntity::dump_segments(const IndexDumper::Pointer &dumper,
key_t *keys) const {
std::vector<node_id_t> n2o_mapping;
std::vector<node_id_t> o2n_mapping;
reshuffle_vectors(&n2o_mapping, &o2n_mapping, keys);
// Remap entry point
VamanaHeader dump_header_copy = header_;
if (dump_header_copy.graph.entry_point != kInvalidNodeId) {
dump_header_copy.graph.entry_point =
o2n_mapping[dump_header_copy.graph.entry_point];
}
int64_t hd_size = dump_header(dumper, dump_header_copy);
if (hd_size < 0) return hd_size;
int64_t vec_size = dump_vectors(dumper, n2o_mapping);
if (vec_size < 0) return vec_size;
int64_t nbr_size = dump_neighbors(dumper, n2o_mapping, o2n_mapping);
if (nbr_size < 0) return nbr_size;
int64_t map_size = dump_mapping_segment(dumper, keys);
if (map_size < 0) return map_size;
int64_t dist_size = dump_neighbor_dists(dumper, n2o_mapping);
if (dist_size < 0) return dist_size;
return hd_size + vec_size + nbr_size + map_size + dist_size;
}
} // namespace core
} // namespace zvec
+352
View File
@@ -0,0 +1,352 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string.h>
#include <functional>
#include <ailego/utility/memory_helper.h>
#include <zvec/ailego/container/heap.h>
#include <zvec/ailego/logger/logger.h>
#include <zvec/core/framework/index_dumper.h>
#include <zvec/core/framework/index_error.h>
#include <zvec/core/framework/index_storage.h>
// Reuse typed MemoryBlock and NeighborsT from hnsw_entity.h
#include "algorithm/hnsw/hnsw_entity.h"
namespace zvec {
namespace core {
// Vamana graph header — single-layer graph (no hierarchical levels)
struct VamanaGraphHeader {
uint32_t size;
uint32_t version;
uint32_t graph_type;
uint32_t doc_count;
uint32_t vector_size;
uint32_t node_size;
uint32_t max_degree; // R: maximum out-degree
uint32_t search_list_size; // L: search list size for construction
uint32_t max_occlusion_size; // C: max candidate size for RobustPrune
uint32_t entry_point; // medoid node id
uint32_t options;
uint32_t reserved_pad;
float alpha; // alpha parameter for RobustPrune
uint8_t reserved_[4076];
};
static_assert(sizeof(VamanaGraphHeader) % 32 == 0,
"VamanaGraphHeader must be aligned with 32 bytes");
struct VamanaHeader {
VamanaHeader() {
clear();
}
VamanaHeader(const VamanaHeader &header) {
memcpy(this, &header, sizeof(header));
}
VamanaHeader &operator=(const VamanaHeader &header) {
memcpy(this, &header, sizeof(header));
return *this;
}
void inline reset() {
graph.doc_count = 0U;
graph.entry_point = kInvalidNodeId;
}
void inline clear() {
memset(this, 0, sizeof(VamanaHeader));
graph.entry_point = kInvalidNodeId;
graph.size = sizeof(VamanaGraphHeader);
graph.alpha = 1.2f;
}
size_t max_degree() const {
return graph.max_degree;
}
size_t vector_size() const {
return graph.vector_size;
}
size_t search_list_size() const {
return graph.search_list_size;
}
size_t max_occlusion_size() const {
return graph.max_occlusion_size;
}
float alpha() const {
return graph.alpha;
}
node_id_t entry_point() const {
return graph.entry_point;
}
node_id_t doc_cnt() const {
return graph.doc_count;
}
VamanaGraphHeader graph;
};
// VamanaEntity: base class for Vamana graph data management
class VamanaEntity {
public:
VamanaEntity() {}
VamanaEntity(const VamanaHeader &hd) {
header_ = hd;
}
virtual ~VamanaEntity() {}
typedef std::shared_ptr<VamanaEntity> Pointer;
// Options bit flags (stored in VamanaGraphHeader::options)
static constexpr uint32_t kOptionSaturateGraph = 1U << 0;
// Default constants
static constexpr uint32_t kDefaultMaxDegree = 64;
static constexpr uint32_t kDefaultSearchListSize = 100;
static constexpr uint32_t kDefaultMaxOcclusionSize = 750;
static constexpr float kDefaultAlpha = 1.2f;
static constexpr bool kDefaultSaturateGraph = false;
static constexpr uint32_t kDefaultEf = 200;
static constexpr float kDefaultScanRatio = 0.1f;
static constexpr uint32_t kDefaultBruteForceThreshold = 1000U;
static constexpr uint32_t kDefaultDocsHardLimit = 1 << 30U;
static constexpr float kDefaultDocsSoftLimitRatio = 0.9f;
static constexpr size_t kMaxChunkSize = 0xFFFFFFFF;
static constexpr size_t kDefaultChunkSize = 2UL * 1024UL * 1024UL;
static constexpr size_t kDefaultMaxChunkCnt = 50000UL;
static constexpr uint32_t kDefaultMinScanLimit = 10000;
static constexpr uint32_t kDefaultMaxScanLimit =
std::numeric_limits<uint32_t>::max();
static constexpr float kDefaultBFNegativeProbability = 0.001f;
inline size_t max_degree() const {
return header_.graph.max_degree;
}
inline node_id_t *mutable_doc_cnt() {
return &header_.graph.doc_count;
}
inline node_id_t doc_cnt() const {
return header_.graph.doc_count;
}
inline float alpha() const {
return header_.graph.alpha;
}
inline size_t search_list_size() const {
return header_.graph.search_list_size;
}
inline size_t max_occlusion_size() const {
return header_.graph.max_occlusion_size;
}
inline node_id_t entry_point() const {
return header_.graph.entry_point;
}
inline size_t vector_size() const {
return header_.graph.vector_size;
}
inline size_t node_size() const {
return header_.graph.node_size;
}
void set_vector_size(size_t size) {
header_.graph.vector_size = size;
}
void set_max_degree(uint32_t val) {
header_.graph.max_degree = val;
}
void set_search_list_size(uint32_t val) {
header_.graph.search_list_size = val;
}
void set_max_occlusion_size(uint32_t val) {
header_.graph.max_occlusion_size = val;
}
void set_alpha(float val) {
header_.graph.alpha = val;
}
inline bool saturate_graph() const {
return (header_.graph.options & kOptionSaturateGraph) != 0;
}
void set_saturate_graph(bool val) {
if (val) {
header_.graph.options |= kOptionSaturateGraph;
} else {
header_.graph.options &= ~kOptionSaturateGraph;
}
}
// Neighbor size: NeighborsHeader + max_degree * sizeof(node_id_t)
inline size_t neighbors_size() const {
return sizeof(NeighborsHeader) + max_degree() * sizeof(node_id_t);
}
virtual void update_entry_point(node_id_t ep) {
header_.graph.entry_point = ep;
}
virtual int cleanup() {
header_.clear();
return 0;
}
virtual const VamanaEntity::Pointer clone() const {
return VamanaEntity::Pointer();
}
// Pure virtual interface
virtual key_t get_key(node_id_t id) const = 0;
virtual const void *get_vector(node_id_t id) const = 0;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const = 0;
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const = 0;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const = 0;
virtual const Neighbors get_neighbors(node_id_t id) const = 0;
virtual int add_vector(key_t key, const void *vec, node_id_t *id) {
return IndexError_NotImplemented;
}
virtual int add_vector_with_id(node_id_t id, const void *vec) {
return IndexError_NotImplemented;
}
virtual int update_neighbors(
node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
return IndexError_NotImplemented;
}
virtual void add_neighbor(node_id_t id, uint32_t size,
node_id_t neighbor_id) {}
// --- Neighbor distance storage (CSR-like, lazy-loaded) ---
// Each node has max_degree dist_t slots, the i-th slot stores the distance
// from this node to its i-th neighbor. Only allocated/loaded when needed
// (first write operation). Search-only paths never touch this data.
// Ensure distance storage is allocated/loaded. Must be called before
// any get/set neighbor dist operations. Thread-safe (idempotent).
virtual int ensure_dist_storage() {
return 0;
}
// Whether distance storage is currently loaded
virtual bool dist_storage_loaded() const {
return false;
}
// Get pointer to the distance array for node `id`.
// Returns nullptr if dist storage is not loaded.
virtual const dist_t *get_neighbor_dists(node_id_t id) const {
return nullptr;
}
// Update all neighbor distances for node `id` from a prune result.
virtual void update_neighbor_dists(
node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {}
// Set the distance for the `idx`-th neighbor of node `id`.
virtual void set_neighbor_dist(node_id_t id, uint32_t idx, dist_t dist) {}
virtual int dump(const IndexDumper::Pointer &dumper) {
return IndexError_NotImplemented;
}
virtual const void *get_vector_by_key(uint64_t /*key*/) const {
return nullptr;
}
virtual int get_vector_by_key(const key_t /*key*/,
IndexStorage::MemoryBlock & /*block*/) const {
return IndexError_NotImplemented;
}
static int CalcAndAddPadding(const IndexDumper::Pointer &dumper,
size_t data_size, size_t *padding_size);
protected:
inline const VamanaHeader &header() const {
return header_;
}
inline VamanaHeader *mutable_header() {
return &header_;
}
inline size_t header_size() const {
return sizeof(header_);
}
void set_node_size(size_t size) {
header_.graph.node_size = size;
}
int64_t dump_segments(const IndexDumper::Pointer &dumper, key_t *keys) const;
int64_t dump_segment(const IndexDumper::Pointer &dumper,
const std::string &segment_id, const void *data,
size_t size) const;
int64_t dump_header(const IndexDumper::Pointer &dumper,
const VamanaHeader &hd) const;
int64_t dump_vectors(const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping) const;
int64_t dump_neighbors(const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping,
const std::vector<node_id_t> &neighbor_mapping) const;
int64_t dump_neighbor_dists(
const IndexDumper::Pointer &dumper,
const std::vector<node_id_t> &reorder_mapping) const;
int64_t dump_mapping_segment(const IndexDumper::Pointer &dumper,
const key_t *keys) const;
void reshuffle_vectors(std::vector<node_id_t> *n2o_mapping,
std::vector<node_id_t> *o2n_mapping,
key_t *keys) const;
static inline size_t AlignSize(size_t size) {
return (size + 0x1F) & (~0x1F);
}
static inline size_t AlignPageSize(size_t size) {
size_t page_mask = ailego::MemoryHelper::PageSize() - 1;
return (size + page_mask) & (~page_mask);
}
static inline size_t AlignHugePageSize(size_t size) {
size_t page_mask = ailego::MemoryHelper::HugePageSize() - 1;
return (size + page_mask) & (~page_mask);
}
public:
const static std::string kGraphHeaderSegmentId;
const static std::string kGraphFeaturesSegmentId;
const static std::string kGraphKeysSegmentId;
const static std::string kGraphNeighborsSegmentId;
const static std::string kGraphOffsetsSegmentId;
const static std::string kGraphMappingSegmentId;
const static std::string kGraphNeighborDistsSegmentId;
static constexpr uint32_t kRevision = 0U;
protected:
VamanaHeader header_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,117 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <zvec/core/framework/index_provider.h>
#include <zvec/core/framework/index_searcher.h>
#include <zvec/core/framework/index_streamer.h>
#include "vamana_entity.h"
namespace zvec {
namespace core {
class VamanaIndexProvider : public IndexProvider {
public:
VamanaIndexProvider(const IndexMeta &meta,
const VamanaEntity::Pointer &entity,
const std::string &owner)
: meta_(meta), entity_(entity), owner_class_(owner) {}
VamanaIndexProvider(const VamanaIndexProvider &) = delete;
VamanaIndexProvider &operator=(const VamanaIndexProvider &) = delete;
public:
IndexProvider::Iterator::Pointer create_iterator() override {
return VamanaIndexProvider::Iterator::Pointer(new (std::nothrow)
Iterator(entity_));
}
size_t count(void) const override {
return entity_->doc_cnt();
}
size_t dimension(void) const override {
return meta_.dimension();
}
IndexMeta::DataType data_type(void) const override {
return meta_.data_type();
}
size_t element_size(void) const override {
return meta_.element_size();
}
const void *get_vector(uint64_t key) const override {
return entity_->get_vector_by_key(key);
}
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector_by_key(key, block);
}
const std::string &owner_class(void) const override {
return owner_class_;
}
private:
class Iterator : public IndexProvider::Iterator {
public:
Iterator(const VamanaEntity::Pointer &entity)
: entity_(entity), cur_id_(0U) {
cur_id_ = get_next_valid_id(0);
}
const void *data(void) const override {
return entity_->get_vector(cur_id_);
}
bool is_valid(void) const override {
return cur_id_ < entity_->doc_cnt();
}
uint64_t key(void) const override {
return entity_->get_key(cur_id_);
}
void next(void) override {
cur_id_ = get_next_valid_id(cur_id_ + 1);
}
void reset(void) {
cur_id_ = get_next_valid_id(0);
}
private:
node_id_t get_next_valid_id(node_id_t start_id) {
for (node_id_t i = start_id; i < entity_->doc_cnt(); i++) {
if (entity_->get_key(i) != kInvalidNodeId) {
return i;
}
}
return kInvalidNodeId;
}
const VamanaEntity::Pointer entity_;
node_id_t cur_id_;
};
const IndexMeta &meta_;
const VamanaEntity::Pointer entity_;
const std::string owner_class_;
};
} // namespace core
} // namespace zvec
+79
View File
@@ -0,0 +1,79 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
namespace zvec {
namespace core {
// Builder parameters
static const std::string PARAM_VAMANA_BUILDER_THREAD_COUNT(
"proxima.vamana.builder.thread_count");
static const std::string PARAM_VAMANA_BUILDER_MEMORY_QUOTA(
"proxima.vamana.builder.memory_quota");
static const std::string PARAM_VAMANA_BUILDER_MAX_DEGREE(
"proxima.vamana.builder.max_degree");
static const std::string PARAM_VAMANA_BUILDER_SEARCH_LIST_SIZE(
"proxima.vamana.builder.search_list_size");
static const std::string PARAM_VAMANA_BUILDER_ALPHA(
"proxima.vamana.builder.alpha");
static const std::string PARAM_VAMANA_BUILDER_MAX_OCCLUSION_SIZE(
"proxima.vamana.builder.max_occlusion_size");
// Searcher parameters
static const std::string PARAM_VAMANA_SEARCHER_SEARCH_LIST_SIZE(
"proxima.vamana.searcher.search_list_size");
static const std::string PARAM_VAMANA_SEARCHER_BRUTE_FORCE_THRESHOLD(
"proxima.vamana.searcher.brute_force_threshold");
// Streamer parameters
static const std::string PARAM_VAMANA_STREAMER_MAX_DEGREE(
"proxima.vamana.streamer.max_degree");
static const std::string PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE(
"proxima.vamana.streamer.search_list_size");
static const std::string PARAM_VAMANA_STREAMER_ALPHA(
"proxima.vamana.streamer.alpha");
static const std::string PARAM_VAMANA_STREAMER_MAX_OCCLUSION_SIZE(
"proxima.vamana.streamer.max_occlusion_size");
static const std::string PARAM_VAMANA_STREAMER_EF("proxima.vamana.streamer.ef");
static const std::string PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD(
"proxima.vamana.streamer.brute_force_threshold");
static const std::string PARAM_VAMANA_STREAMER_MAX_SCAN_RATIO(
"proxima.vamana.streamer.max_scan_ratio");
static const std::string PARAM_VAMANA_STREAMER_DOCS_HARD_LIMIT(
"proxima.vamana.streamer.docs_hard_limit");
static const std::string PARAM_VAMANA_STREAMER_DOCS_SOFT_LIMIT(
"proxima.vamana.streamer.docs_soft_limit");
static const std::string PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE(
"proxima.vamana.streamer.max_index_size");
static const std::string PARAM_VAMANA_STREAMER_CHUNK_SIZE(
"proxima.vamana.streamer.chunk_size");
static const std::string PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE(
"proxima.vamana.streamer.get_vector_enable");
static const std::string PARAM_VAMANA_STREAMER_FORCE_PADDING_RESULT_ENABLE(
"proxima.vamana.streamer.force_padding_result_enable");
static const std::string PARAM_VAMANA_STREAMER_USE_ID_MAP(
"proxima.vamana.streamer.use_id_map");
static const std::string PARAM_VAMANA_STREAMER_MAX_SCAN_LIMIT(
"proxima.vamana.streamer.max_scan_limit");
static const std::string PARAM_VAMANA_STREAMER_MIN_SCAN_LIMIT(
"proxima.vamana.streamer.min_scan_limit");
static const std::string PARAM_VAMANA_STREAMER_SATURATE_GRAPH(
"proxima.vamana.streamer.saturate_graph");
static const std::string PARAM_VAMANA_STREAMER_USE_CONTIGUOUS_MEMORY(
"proxima.vamana.streamer.use_contiguous_memory");
} // namespace core
} // namespace zvec
@@ -0,0 +1,714 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "vamana_streamer.h"
#include <iostream>
#include <ailego/pattern/defer.h>
#include <ailego/utility/memory_helper.h>
#include "vamana_algorithm.h"
#include "vamana_context.h"
#include "vamana_dist_calculator.h"
#include "vamana_index_provider.h"
namespace zvec {
namespace core {
VamanaStreamer::VamanaStreamer() = default;
VamanaStreamer::~VamanaStreamer() {
if (state_ == STATE_INITED) {
this->cleanup();
}
}
int VamanaStreamer::init(const IndexMeta &imeta, const ailego::Params &params) {
meta_ = imeta;
meta_.set_streamer("VamanaStreamer", VamanaEntity::kRevision, params);
params.get(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, &max_index_size_);
params.get(PARAM_VAMANA_STREAMER_MAX_DEGREE, &max_degree_);
params.get(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, &search_list_size_);
params.get(PARAM_VAMANA_STREAMER_ALPHA, &alpha_);
params.get(PARAM_VAMANA_STREAMER_MAX_OCCLUSION_SIZE, &max_occlusion_size_);
params.get(PARAM_VAMANA_STREAMER_EF, &ef_);
params.get(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
params.get(PARAM_VAMANA_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
params.get(PARAM_VAMANA_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
params.get(PARAM_VAMANA_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
params.get(PARAM_VAMANA_STREAMER_CHUNK_SIZE, &chunk_size_);
params.get(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, &get_vector_enabled_);
params.get(PARAM_VAMANA_STREAMER_FORCE_PADDING_RESULT_ENABLE,
&force_padding_topk_enabled_);
params.get(PARAM_VAMANA_STREAMER_USE_ID_MAP, &use_id_map_);
params.get(PARAM_VAMANA_STREAMER_DOCS_HARD_LIMIT, &docs_hard_limit_);
params.get(PARAM_VAMANA_STREAMER_SATURATE_GRAPH, &saturate_graph_);
params.get(PARAM_VAMANA_STREAMER_USE_CONTIGUOUS_MEMORY,
&use_contiguous_memory_);
size_t docs_soft_limit = 0;
params.get(PARAM_VAMANA_STREAMER_DOCS_SOFT_LIMIT, &docs_soft_limit);
if (docs_soft_limit > 0 && docs_soft_limit > docs_hard_limit_) {
LOG_ERROR("[%s] must be >= [%s]",
PARAM_VAMANA_STREAMER_DOCS_HARD_LIMIT.c_str(),
PARAM_VAMANA_STREAMER_DOCS_SOFT_LIMIT.c_str());
return IndexError_InvalidArgument;
} else if (docs_soft_limit == 0UL) {
docs_soft_limit_ =
docs_hard_limit_ * VamanaEntity::kDefaultDocsSoftLimitRatio;
} else {
docs_soft_limit_ = docs_soft_limit;
}
// Validate parameters
if (max_degree_ == 0U) max_degree_ = VamanaEntity::kDefaultMaxDegree;
if (search_list_size_ == 0U)
search_list_size_ = VamanaEntity::kDefaultSearchListSize;
if (max_occlusion_size_ == 0U)
max_occlusion_size_ = VamanaEntity::kDefaultMaxOcclusionSize;
if (alpha_ <= 0.0f) alpha_ = VamanaEntity::kDefaultAlpha;
if (ef_ == 0U) ef_ = VamanaEntity::kDefaultEf;
if (chunk_size_ == 0UL) chunk_size_ = VamanaEntity::kDefaultChunkSize;
if (chunk_size_ > VamanaEntity::kMaxChunkSize) {
LOG_ERROR("[%s] must be < %zu", PARAM_VAMANA_STREAMER_CHUNK_SIZE.c_str(),
VamanaEntity::kMaxChunkSize);
return IndexError_InvalidArgument;
}
if (max_scan_ratio_ <= 0.0f || max_scan_ratio_ > 1.0f) {
LOG_ERROR("[%s] must be in range (0.0f,1.0f]",
PARAM_VAMANA_STREAMER_MAX_SCAN_RATIO.c_str());
return IndexError_InvalidArgument;
}
if (max_scan_limit_ < min_scan_limit_) {
LOG_ERROR("[%s] must be >= [%s]",
PARAM_VAMANA_STREAMER_MAX_SCAN_LIMIT.c_str(),
PARAM_VAMANA_STREAMER_MIN_SCAN_LIMIT.c_str());
return IndexError_InvalidArgument;
}
LOG_DEBUG(
"Vamana init params: maxIndexSize=%zu docsHardLimit=%zu "
"docsSoftLimit=%zu maxDegree=%u searchListSize=%u alpha=%.2f "
"maxOcclusionSize=%u ef=%u maxScanRatio=%.3f minScanLimit=%zu "
"maxScanLimit=%zu bruteForceThreshold=%zu chunkSize=%zu "
"getVectorEnabled=%u forcePadding=%u",
max_index_size_, docs_hard_limit_, docs_soft_limit_, max_degree_,
search_list_size_, alpha_, max_occlusion_size_, ef_, max_scan_ratio_,
min_scan_limit_, max_scan_limit_, bruteforce_threshold_, chunk_size_,
get_vector_enabled_, force_padding_topk_enabled_);
state_ = STATE_INITED;
return 0;
}
int VamanaStreamer::cleanup(void) {
if (state_ == STATE_OPENED) {
this->close();
}
LOG_INFO("VamanaStreamer cleanup");
meta_.clear();
metric_.reset();
stats_.clear();
if (entity_) entity_->cleanup();
if (alg_) alg_->cleanup();
max_index_size_ = 0UL;
docs_hard_limit_ = VamanaEntity::kDefaultDocsHardLimit;
docs_soft_limit_ = 0UL;
max_degree_ = VamanaEntity::kDefaultMaxDegree;
search_list_size_ = VamanaEntity::kDefaultSearchListSize;
max_occlusion_size_ = VamanaEntity::kDefaultMaxOcclusionSize;
alpha_ = VamanaEntity::kDefaultAlpha;
ef_ = VamanaEntity::kDefaultEf;
bruteforce_threshold_ = VamanaEntity::kDefaultBruteForceThreshold;
max_scan_limit_ = VamanaEntity::kDefaultMaxScanLimit;
min_scan_limit_ = VamanaEntity::kDefaultMinScanLimit;
chunk_size_ = VamanaEntity::kDefaultChunkSize;
max_scan_ratio_ = VamanaEntity::kDefaultScanRatio;
state_ = STATE_INIT;
check_crc_enabled_ = false;
get_vector_enabled_ = false;
return 0;
}
int VamanaStreamer::setup_entity() {
entity_->set_use_key_info_map(use_id_map_);
entity_->set_vector_size(meta_.element_size());
entity_->set_chunk_size(chunk_size_);
entity_->set_get_vector(get_vector_enabled_);
// Set Vamana-specific parameters via public setters
entity_->set_max_degree(max_degree_);
entity_->set_search_list_size(search_list_size_);
entity_->set_max_occlusion_size(max_occlusion_size_);
entity_->set_alpha(alpha_);
entity_->set_saturate_graph(saturate_graph_);
int ret = entity_->init(docs_hard_limit_);
if (ret != 0) {
LOG_ERROR("Vamana entity init failed: %s", IndexError::What(ret));
}
return ret;
}
int VamanaStreamer::open(IndexStorage::Pointer stg) {
LOG_INFO("VamanaStreamer open");
if (ailego_unlikely(state_ != STATE_INITED)) {
LOG_ERROR("Open storage failed, init streamer first!");
return IndexError_NoReady;
}
// Create entity based on storage type
switch (stg->memory_block_type()) {
case IndexStorage::MemoryBlock::MBT_BUFFERPOOL: {
entity_ = std::make_unique<VamanaBufferPoolStreamerEntity>(stats_);
break;
}
default: {
if (use_contiguous_memory_) {
entity_ = std::make_unique<VamanaContiguousStreamerEntity>(stats_);
} else {
entity_ = std::make_unique<VamanaMmapStreamerEntity>(stats_);
}
break;
}
}
auto cleanup_on_error = [this]() {
if (entity_) {
entity_->close();
entity_.reset();
}
alg_.reset();
metric_.reset();
};
int ret = setup_entity();
if (ret != 0) {
cleanup_on_error();
return ret;
}
ret = entity_->open(std::move(stg), max_index_size_, check_crc_enabled_);
if (ret != 0) {
cleanup_on_error();
return ret;
}
// Handle IndexMeta
IndexMeta index_meta;
ret = entity_->get_index_meta(&index_meta);
if (ret == IndexError_NoExist) {
ret = entity_->set_index_meta(meta_);
if (ret != 0) {
LOG_ERROR("Failed to set index meta: %s", IndexError::What(ret));
cleanup_on_error();
return ret;
}
} else if (ret != 0) {
LOG_ERROR("Failed to get index meta: %s", IndexError::What(ret));
cleanup_on_error();
return ret;
} else {
if (index_meta.dimension() != meta_.dimension() ||
index_meta.element_size() != meta_.element_size() ||
index_meta.metric_name() != meta_.metric_name() ||
index_meta.data_type() != meta_.data_type()) {
LOG_ERROR("IndexMeta mismatch from the previous in index");
cleanup_on_error();
return IndexError_Mismatch;
}
auto metric_params = index_meta.metric_params();
metric_params.merge(meta_.metric_params());
meta_.set_metric(index_meta.metric_name(), 0, metric_params);
}
// Create metric
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
if (!metric_) {
LOG_ERROR("Failed to create metric %s", meta_.metric_name().c_str());
cleanup_on_error();
return IndexError_NoExist;
}
ret = metric_->init(meta_, meta_.metric_params());
if (ret != 0) {
LOG_ERROR("Failed to init metric, ret=%d", ret);
cleanup_on_error();
return ret;
}
if (!metric_->distance() || !metric_->batch_distance()) {
LOG_ERROR("Invalid metric distance functions");
cleanup_on_error();
return IndexError_InvalidArgument;
}
add_distance_ = metric_->distance();
add_batch_distance_ = metric_->batch_distance();
search_distance_ = add_distance_;
search_batch_distance_ = add_batch_distance_;
if (metric_->query_metric() && metric_->query_metric()->distance() &&
metric_->query_metric()->batch_distance()) {
search_distance_ = metric_->query_metric()->distance();
search_batch_distance_ = metric_->query_metric()->batch_distance();
}
// Create algorithm based on entity storage mode
switch (entity_->storage_mode()) {
case VamanaStorageMode::kBufferPool:
alg_ = VamanaAlgorithmBase::UPointer(
new VamanaAlgorithm<VamanaBufferPoolStreamerEntity>(
static_cast<VamanaBufferPoolStreamerEntity &>(*entity_)));
break;
case VamanaStorageMode::kContiguous: {
auto &contiguous_entity =
static_cast<VamanaContiguousStreamerEntity &>(*entity_);
int build_ret = contiguous_entity.build_contiguous_memory();
if (build_ret != 0) {
LOG_ERROR("Failed to build contiguous memory, ret=%d", build_ret);
cleanup_on_error();
return build_ret;
}
alg_ = VamanaAlgorithmBase::UPointer(
new VamanaAlgorithm<VamanaContiguousStreamerEntity>(
contiguous_entity));
break;
}
default:
alg_ = VamanaAlgorithmBase::UPointer(
new VamanaAlgorithm<VamanaMmapStreamerEntity>(
static_cast<VamanaMmapStreamerEntity &>(*entity_)));
break;
}
ret = alg_->init();
if (ret != 0) {
cleanup_on_error();
return ret;
}
state_ = STATE_OPENED;
magic_ = IndexContext::GenerateMagic();
return 0;
}
int VamanaStreamer::close(void) {
LOG_INFO("VamanaStreamer close");
stats_.clear();
meta_.set_metric(metric_->name(), 0, metric_->params());
entity_->set_index_meta(meta_);
int ret = entity_->close();
if (ret != 0) return ret;
state_ = STATE_INITED;
return 0;
}
int VamanaStreamer::flush(uint64_t checkpoint) {
LOG_INFO("VamanaStreamer flush checkpoint=%zu", (size_t)checkpoint);
meta_.set_metric(metric_->name(), 0, metric_->params());
entity_->set_index_meta(meta_);
return entity_->flush(checkpoint);
}
int VamanaStreamer::dump(const IndexDumper::Pointer &dumper) {
LOG_INFO("VamanaStreamer dump");
shared_mutex_.lock();
AILEGO_DEFER([&]() { shared_mutex_.unlock(); });
meta_.set_searcher("VamanaSearcher", VamanaEntity::kRevision,
ailego::Params());
int ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
if (ret != 0) {
LOG_ERROR("Failed to serialize meta into dumper.");
return ret;
}
return entity_->dump(dumper);
}
IndexStreamer::Context::Pointer VamanaStreamer::create_context(void) const {
if (ailego_unlikely(state_ != STATE_OPENED)) {
LOG_ERROR("Create context failed, open storage first!");
return Context::Pointer();
}
VamanaEntity::Pointer entity = entity_->clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("CreateContext clone failed");
return Context::Pointer();
}
VamanaContext *ctx =
new (std::nothrow) VamanaContext(meta_.dimension(), metric_, entity);
if (ailego_unlikely(ctx == nullptr)) {
LOG_ERROR("Failed to new VamanaContext");
return Context::Pointer();
}
ctx->set_ef(ef_);
ctx->set_max_scan_limit(max_scan_limit_);
ctx->set_min_scan_limit(min_scan_limit_);
ctx->set_max_scan_ratio(max_scan_ratio_);
ctx->set_magic(magic_);
ctx->set_force_padding_topk(force_padding_topk_enabled_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
if (ailego_unlikely(ctx->init(VamanaContext::kStreamerContext) != 0)) {
LOG_ERROR("Init VamanaContext failed");
delete ctx;
return Context::Pointer();
}
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
return Context::Pointer(ctx);
}
IndexProvider::Pointer VamanaStreamer::create_provider(void) const {
LOG_DEBUG("VamanaStreamer create provider");
auto entity = entity_->clone();
if (ailego_unlikely(!entity)) {
LOG_ERROR("Clone VamanaEntity failed");
return nullptr;
}
return IndexProvider::Pointer(
new VamanaIndexProvider(meta_, entity, "VamanaStreamer"));
}
int VamanaStreamer::update_context(VamanaContext *ctx) const {
const VamanaEntity::Pointer entity = entity_->clone();
if (!entity) {
LOG_ERROR("Failed to clone search context entity");
return IndexError_Runtime;
}
ctx->set_max_scan_limit(max_scan_limit_);
ctx->set_min_scan_limit(min_scan_limit_);
ctx->set_max_scan_ratio(max_scan_ratio_);
ctx->set_bruteforce_threshold(bruteforce_threshold_);
return ctx->update_context(VamanaContext::kStreamerContext, meta_, metric_,
entity, magic_);
}
int VamanaStreamer::add_impl(uint64_t pkey, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) return ret;
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to VamanaContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
ret = update_context(ctx);
if (ret != 0) return ret;
}
if (ailego_unlikely(entity_->doc_cnt() >= docs_soft_limit_)) {
if (entity_->doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %u exceed hard limit", entity_->doc_cnt());
const std::lock_guard<std::mutex> lk(mutex_);
(*stats_.mutable_discarded_count())++;
return IndexError_IndexFull;
} else {
LOG_WARN("Current docs %u exceed soft limit", entity_->doc_cnt());
}
}
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
LOG_ERROR("Cannot add vector while dumping index");
(*stats_.mutable_discarded_count())++;
return IndexError_Unsupported;
}
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
if (metric_->support_train()) {
const std::lock_guard<std::mutex> lk(mutex_);
ret = metric_->train(query, meta_.dimension());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Vamana streamer metric train failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
}
node_id_t id;
ret = entity_->add_vector(pkey, query, &id);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Vamana streamer add vector failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
ret = alg_->add_node(id, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Vamana streamer add node failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
if (ailego_unlikely(ctx->error())) {
(*stats_.mutable_discarded_count())++;
return IndexError_Runtime;
}
(*stats_.mutable_added_count())++;
return 0;
}
int VamanaStreamer::add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) return ret;
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to VamanaContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
ret = update_context(ctx);
if (ret != 0) return ret;
}
if (ailego_unlikely(entity_->doc_cnt() >= docs_soft_limit_)) {
if (entity_->doc_cnt() >= docs_hard_limit_) {
LOG_ERROR("Current docs %u exceed hard limit", entity_->doc_cnt());
const std::lock_guard<std::mutex> lk(mutex_);
(*stats_.mutable_discarded_count())++;
return IndexError_IndexFull;
} else {
LOG_WARN("Current docs %u exceed soft limit", entity_->doc_cnt());
}
}
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
LOG_ERROR("Cannot add vector while dumping index");
(*stats_.mutable_discarded_count())++;
return IndexError_Unsupported;
}
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
if (metric_->support_train()) {
const std::lock_guard<std::mutex> lk(mutex_);
ret = metric_->train(query, meta_.dimension());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Vamana streamer metric train failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
}
ret = entity_->add_vector_with_id(id, query);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Vamana streamer add vector failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
ret = alg_->add_node(id, ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Vamana streamer add node failed");
(*stats_.mutable_discarded_count())++;
return ret;
}
if (ailego_unlikely(ctx->error())) {
(*stats_.mutable_discarded_count())++;
return IndexError_Runtime;
}
(*stats_.mutable_added_count())++;
return 0;
}
int VamanaStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const {
return search_impl(query, qmeta, 1, context);
}
int VamanaStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) return ret;
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to VamanaContext failed");
return IndexError_Cast;
}
if (entity_->doc_cnt() <= ctx->get_bruteforce_threshold()) {
return search_bf_impl(query, qmeta, count, context);
}
if (ctx->magic() != magic_) {
ret = update_context(ctx);
if (ret != 0) return ret;
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
for (size_t q = 0; q < count; ++q) {
ctx->reset_query(query);
ret = alg_->search(ctx);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Vamana search failed");
return ret;
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) return IndexError_Runtime;
return 0;
}
void VamanaStreamer::print_debug_info() {
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
if (entity_->get_key(id) == kInvalidKey) continue;
Neighbors neighbours = entity_->get_neighbors(id);
std::cout << "node: " << id << "; ";
if (neighbours.size() == 0) {
std::cout << std::endl;
continue;
}
for (uint32_t i = 0; i < neighbours.size(); ++i) {
std::cout << neighbours[i];
if (i == neighbours.size() - 1) {
std::cout << std::endl;
} else {
std::cout << ", ";
}
}
}
}
int VamanaStreamer::search_bf_impl(const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) const {
return search_bf_impl(query, qmeta, 1, context);
}
int VamanaStreamer::search_bf_impl(const void *query,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) return ret;
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to VamanaContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
ret = update_context(ctx);
if (ret != 0) return ret;
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
const auto &filter = static_cast<IndexContext *>(ctx)->filter();
auto &topk = ctx->topk_heap();
for (size_t q = 0; q < count; ++q) {
ctx->reset_query(query);
topk.clear();
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
if (entity_->get_key(id) == kInvalidKey) continue;
if (!filter.is_valid() || !filter(entity_->get_key(id))) {
dist_t dist = ctx->dist_calculator().batch_dist(id);
topk.emplace(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) return IndexError_Runtime;
return 0;
}
int VamanaStreamer::search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
int ret = check_params(query, qmeta);
if (ailego_unlikely(ret != 0)) return ret;
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
ailego_do_if_false(ctx) {
LOG_ERROR("Cast context to VamanaContext failed");
return IndexError_Cast;
}
if (ctx->magic() != magic_) {
ret = update_context(ctx);
if (ret != 0) return ret;
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
auto &topk = ctx->topk_heap();
for (size_t q = 0; q < count; ++q) {
ctx->reset_query(query);
topk.clear();
for (const auto &keys : p_keys) {
for (auto key : keys) {
node_id_t id = entity_->get_id(key);
if (id == kInvalidNodeId) continue;
dist_t dist = ctx->dist_calculator().batch_dist(id);
topk.emplace(id, dist);
}
}
ctx->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
if (ailego_unlikely(ctx->error())) return IndexError_Runtime;
return 0;
}
INDEX_FACTORY_REGISTER_STREAMER(VamanaStreamer);
} // namespace core
} // namespace zvec
+192
View File
@@ -0,0 +1,192 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/parallel/lock.h>
#include <zvec/core/framework/index_framework.h>
#include "vamana_algorithm.h"
#include "vamana_streamer_entity.h"
namespace zvec {
namespace core {
class VamanaStreamer : public IndexStreamer {
public:
using ContextPointer = IndexStreamer::Context::Pointer;
VamanaStreamer(void);
virtual ~VamanaStreamer(void);
VamanaStreamer(const VamanaStreamer &) = delete;
VamanaStreamer &operator=(const VamanaStreamer &) = delete;
protected:
virtual int init(const IndexMeta &imeta,
const ailego::Params &params) override;
virtual int cleanup(void) override;
virtual Context::Pointer create_context(void) const override;
virtual IndexProvider::Pointer create_provider(void) const override;
virtual int add_impl(uint64_t pkey, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
virtual int add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
virtual const void *get_vector(uint64_t key) const override {
return entity_->get_vector_by_key(key);
}
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector_by_key(key, block);
}
virtual const void *get_vector_by_id(uint32_t id) const override {
return entity_->get_vector(id);
}
virtual int get_vector_by_id(
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector(id, block);
}
virtual int open(IndexStorage::Pointer stg) override;
virtual int close(void) override;
virtual int flush(uint64_t checkpoint) override;
virtual int dump(const IndexDumper::Pointer &dumper) override;
virtual const Stats &stats(void) const override {
return stats_;
}
virtual const IndexMeta &meta(void) const override {
return meta_;
}
virtual void print_debug_info() override;
private:
inline int check_params(const void *query,
const IndexQueryMeta &qmeta) const {
if (ailego_unlikely(!query)) {
LOG_ERROR("null query");
return IndexError_InvalidArgument;
}
if (ailego_unlikely(qmeta.dimension() != meta_.dimension() ||
qmeta.data_type() != meta_.data_type() ||
qmeta.element_size() != meta_.element_size())) {
LOG_ERROR("Unsupported query meta");
return IndexError_Mismatch;
}
return 0;
}
int setup_entity();
int update_context(VamanaContext *ctx) const;
private:
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_OPENED = 2 };
class Stats : public IndexStreamer::Stats {
public:
void clear(void) {
set_revision_id(0u);
set_loaded_count(0u);
set_added_count(0u);
set_discarded_count(0u);
set_index_size(0u);
set_dumped_size(0u);
set_check_point(0u);
set_create_time(0u);
set_update_time(0u);
clear_attributes();
}
};
std::unique_ptr<VamanaStreamerEntity> entity_;
VamanaAlgorithmBase::UPointer alg_;
IndexMeta meta_{};
IndexMetric::Pointer metric_{};
IndexMetric::MatrixDistance add_distance_{};
IndexMetric::MatrixDistance search_distance_{};
IndexMetric::MatrixBatchDistance add_batch_distance_{};
IndexMetric::MatrixBatchDistance search_batch_distance_{};
Stats stats_{};
std::mutex mutex_{};
size_t max_index_size_{0UL};
size_t chunk_size_{VamanaEntity::kDefaultChunkSize};
size_t docs_hard_limit_{VamanaEntity::kDefaultDocsHardLimit};
size_t docs_soft_limit_{0UL};
uint32_t max_degree_{VamanaEntity::kDefaultMaxDegree};
uint32_t search_list_size_{VamanaEntity::kDefaultSearchListSize};
uint32_t max_occlusion_size_{VamanaEntity::kDefaultMaxOcclusionSize};
float alpha_{VamanaEntity::kDefaultAlpha};
uint32_t ef_{VamanaEntity::kDefaultEf};
size_t bruteforce_threshold_{VamanaEntity::kDefaultBruteForceThreshold};
size_t max_scan_limit_{VamanaEntity::kDefaultMaxScanLimit};
size_t min_scan_limit_{VamanaEntity::kDefaultMinScanLimit};
float bf_negative_prob_{VamanaEntity::kDefaultBFNegativeProbability};
float max_scan_ratio_{VamanaEntity::kDefaultScanRatio};
uint32_t magic_{0U};
State state_{STATE_INIT};
bool check_crc_enabled_{false};
bool get_vector_enabled_{false};
bool force_padding_topk_enabled_{false};
bool use_id_map_{true};
bool saturate_graph_{VamanaEntity::kDefaultSaturateGraph};
bool use_contiguous_memory_{false};
ailego::SharedMutex shared_mutex_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,773 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "vamana_streamer_entity.h"
#if defined(__linux__) || defined(__APPLE__)
#include <sys/mman.h>
#endif
#include <ailego/utility/memory_helper.h>
#include <zvec/ailego/hash/crc32c.h>
#include <zvec/core/framework/index_stats.h>
namespace zvec {
namespace core {
VamanaStreamerEntity::VamanaStreamerEntity(IndexStreamer::Stats &stats)
: stats_(stats) {
keys_map_lock_ = std::make_shared<ailego::SharedMutex>();
keys_map_ = std::make_shared<HashMap<key_t, node_id_t>>();
keys_map_->set_empty_key(kInvalidKey);
broker_ = std::make_shared<ChunkBroker>(stats);
}
VamanaStreamerEntity::~VamanaStreamerEntity() {}
int VamanaStreamerEntity::cleanup() {
node_chunks_.clear();
if (keys_map_) {
keys_map_->clear();
}
header_.clear();
return 0;
}
int VamanaStreamerEntity::init(size_t /*max_doc_cnt*/) {
// node_size = vector_size + key_size + neighbors_size
set_node_size(vector_size() + sizeof(key_t) + neighbors_size());
neighbor_size_ = neighbors_size();
return 0;
}
key_t VamanaStreamerEntity::get_key(node_id_t id) const {
if (!use_key_info_map_) return id;
auto loc = get_key_chunk_loc(id);
if (ailego_unlikely(loc.first >= node_chunks_.size())) return kInvalidKey;
const void *ptr = nullptr;
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, sizeof(key_t));
if (ailego_unlikely(ret != sizeof(key_t))) {
LOG_ERROR("Read key failed, ret=%zu", ret);
return kInvalidKey;
}
return *reinterpret_cast<const key_t *>(ptr);
}
const void *VamanaStreamerEntity::get_vector(node_id_t id) const {
auto loc = get_vector_chunk_loc(id);
if (ailego_unlikely(loc.first >= node_chunks_.size())) return nullptr;
const void *ptr = nullptr;
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, vector_size());
if (ailego_unlikely(ret != vector_size())) {
LOG_ERROR("Read vector failed, ret=%zu", ret);
return nullptr;
}
return ptr;
}
int VamanaStreamerEntity::get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const {
auto loc = get_vector_chunk_loc(id);
if (ailego_unlikely(loc.first >= node_chunks_.size()))
return IndexError_NoExist;
size_t ret = node_chunks_[loc.first]->read(loc.second, block, vector_size());
if (ailego_unlikely(ret != vector_size())) {
LOG_ERROR("Read vector failed, ret=%zu", ret);
return IndexError_ReadData;
}
return 0;
}
int VamanaStreamerEntity::get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const {
for (uint32_t i = 0; i < count; ++i) {
vecs[i] = get_vector(ids[i]);
if (ailego_unlikely(vecs[i] == nullptr)) {
return IndexError_NoExist;
}
}
return 0;
}
int VamanaStreamerEntity::get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (uint32_t i = 0; i < count; ++i) {
int ret = get_vector(ids[i], vec_blocks[i]);
if (ailego_unlikely(ret != 0)) return ret;
}
return 0;
}
const Neighbors VamanaStreamerEntity::get_neighbors(node_id_t id) const {
auto loc = get_neighbor_chunk_loc(id);
IndexStorage::MemoryBlock mem_block;
size_t ret = loc.first->read(loc.second, mem_block, neighbor_size_);
if (ailego_unlikely(ret != neighbor_size_)) {
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
return Neighbors();
}
return Neighbors(mem_block);
}
int VamanaStreamerEntity::add_vector(key_t key, const void *vec,
node_id_t *id) {
Chunk::Pointer node_chunk;
size_t chunk_offset = static_cast<size_t>(-1);
std::lock_guard<std::mutex> lock(mutex_);
node_id_t local_id = static_cast<node_id_t>(doc_cnt());
uint32_t chunk_index = node_chunks_.size() - 1U;
if (chunk_index == static_cast<uint32_t>(-1) ||
(node_chunks_[chunk_index]->data_size() >=
node_cnt_per_chunk_ * node_size())) {
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
LOG_ERROR("add vector failed for no memory quota");
return IndexError_IndexFull;
}
chunk_index++;
if (auto dret = ensure_dist_chunk_for(chunk_index); dret != 0) {
return dret;
}
auto p = broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_NODE, chunk_index,
chunk_size_);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return p.first;
}
node_chunk = p.second;
chunk_offset = 0UL;
node_chunks_.emplace_back(node_chunk);
} else {
node_chunk = node_chunks_[chunk_index];
chunk_offset = node_chunk->data_size();
}
// Write vector
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
if (ailego_unlikely(size != vector_size())) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
// Write key
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
if (ailego_unlikely(size != sizeof(key_t))) {
LOG_ERROR("Chunk write key failed, ret=%zu", size);
return IndexError_WriteData;
}
// Neighbors are initialized to zero by default (chunk is zero-filled)
chunk_offset += node_size();
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
return IndexError_Runtime;
}
if (use_key_info_map_) {
keys_map_lock_->lock();
(*keys_map_)[key] = local_id;
keys_map_lock_->unlock();
}
*mutable_doc_cnt() += 1;
broker_->mark_dirty();
*id = local_id;
return 0;
}
int VamanaStreamerEntity::add_vector_with_id(node_id_t id, const void *vec) {
Chunk::Pointer node_chunk;
size_t chunk_offset = static_cast<size_t>(-1);
key_t key = id;
std::lock_guard<std::mutex> lock(mutex_);
auto func_get_node_chunk_and_offset = [&](node_id_t node_id) -> int {
uint32_t chunk_idx = node_id >> node_index_mask_bits_;
ailego_assert_with(chunk_idx <= node_chunks_.size(), "invalid chunk idx");
if (chunk_idx == node_chunks_.size()) {
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
LOG_ERROR("add vector failed for no memory quota");
return IndexError_IndexFull;
}
if (auto dret = ensure_dist_chunk_for(chunk_idx); dret != 0) {
return dret;
}
auto p = broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
chunk_size_);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc data chunk failed");
return p.first;
}
node_chunk = p.second;
node_chunks_.emplace_back(node_chunk);
}
node_chunk = node_chunks_[chunk_idx];
chunk_offset = (node_id & node_index_mask_) * node_size();
return 0;
};
// Fill gaps with invalid keys
for (size_t start_id = doc_cnt(); start_id < id; ++start_id) {
if (auto ret = func_get_node_chunk_and_offset(start_id); ret != 0) {
return ret;
}
size_t size = node_chunk->write(chunk_offset + vector_size(), &kInvalidKey,
sizeof(key_t));
if (ailego_unlikely(size != sizeof(key_t))) {
LOG_ERROR("Chunk write key failed, ret=%zu", size);
return IndexError_WriteData;
}
chunk_offset += node_size();
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
return IndexError_Runtime;
}
}
if (auto ret = func_get_node_chunk_and_offset(id); ret != 0) {
return ret;
}
// Write vector
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
if (ailego_unlikely(size != vector_size())) {
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
return IndexError_WriteData;
}
// Write key
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
if (ailego_unlikely(size != sizeof(key_t))) {
LOG_ERROR("Chunk write key failed, ret=%zu", size);
return IndexError_WriteData;
}
if (*mutable_doc_cnt() <= id) {
*mutable_doc_cnt() = id + 1;
chunk_offset += node_size();
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
return IndexError_Runtime;
}
}
if (use_key_info_map_) {
keys_map_lock_->lock();
(*keys_map_)[key] = id;
keys_map_lock_->unlock();
}
broker_->mark_dirty();
return 0;
}
int VamanaStreamerEntity::update_neighbors(
node_id_t id, const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
auto loc = get_neighbor_chunk_loc(id);
uint32_t count = std::min(static_cast<uint32_t>(neighbors.size()),
static_cast<uint32_t>(max_degree()));
// Build neighbor data in a local buffer
size_t nbr_size = neighbors_size();
std::vector<uint8_t> buffer(nbr_size, 0);
auto *hd = reinterpret_cast<NeighborsHeader *>(buffer.data());
hd->neighbor_cnt = count;
for (uint32_t i = 0; i < count; ++i) {
hd->neighbors[i] = neighbors[i].first;
}
size_t ret = loc.first->write(loc.second, buffer.data(), nbr_size);
if (ailego_unlikely(ret != nbr_size)) {
LOG_ERROR("Write neighbors failed, ret=%zu", ret);
return IndexError_WriteData;
}
return 0;
}
void VamanaStreamerEntity::add_neighbor(node_id_t id, uint32_t size,
node_id_t neighbor_id) {
auto loc = get_neighbor_chunk_loc(id);
if (size >= max_degree()) return;
// Read current neighbors
IndexStorage::MemoryBlock mem_block;
size_t ret = loc.first->read(loc.second, mem_block, neighbor_size_);
if (ailego_unlikely(ret != neighbor_size_)) {
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
return;
}
// Copy to mutable buffer, update, and write back
std::vector<uint8_t> buffer(neighbor_size_);
memcpy(buffer.data(), mem_block.data(), neighbor_size_);
auto *hd = reinterpret_cast<NeighborsHeader *>(buffer.data());
hd->neighbors[size] = neighbor_id;
hd->neighbor_cnt = size + 1;
ret = loc.first->write(loc.second, buffer.data(), neighbor_size_);
if (ailego_unlikely(ret != neighbor_size_)) {
LOG_ERROR("Write neighbor failed, ret=%zu", ret);
}
}
void VamanaStreamerEntity::update_entry_point(node_id_t ep) {
VamanaEntity::update_entry_point(ep);
flush_header();
}
int VamanaStreamerEntity::open(IndexStorage::Pointer stg,
uint64_t max_index_size, bool check_crc) {
std::lock_guard<std::mutex> lock(mutex_);
bool huge_page = stg->isHugePage();
int ret = broker_->open(std::move(stg), chunk_size_, check_crc);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Open index failed: %s", IndexError::What(ret));
return ret;
}
ret = init_chunk_params(max_index_size, huge_page);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("init_chunk_params failed: %s", IndexError::What(ret));
return ret;
}
broker_->set_max_chunks_size(max_index_size_);
// Init header
auto header_chunk = broker_->get_chunk(ChunkBroker::CHUNK_TYPE_HEADER,
ChunkBroker::kDefaultChunkSeqId);
if (!header_chunk) {
// Open empty index, create header
auto p =
broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_HEADER,
ChunkBroker::kDefaultChunkSeqId, header_size());
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc header chunk failed");
return p.first;
}
size_t size = p.second->write(0UL, &header(), header_size());
if (ailego_unlikely(size != header_size())) {
LOG_ERROR("Write header chunk failed");
return IndexError_WriteData;
}
return 0;
}
// Open existing index
ret = init_chunks(header_chunk);
if (ailego_unlikely(ret != 0)) return ret;
// Verify total docs
node_id_t total_vecs = 0;
if (!node_chunks_.empty()) {
size_t last_idx = node_chunks_.size() - 1;
if (node_chunks_[last_idx]->data_size() % node_size()) {
LOG_WARN("The index may be broken");
return IndexError_InvalidFormat;
}
total_vecs = last_idx * node_cnt_per_chunk_ +
node_chunks_[last_idx]->data_size() / node_size();
}
LOG_INFO("Open Vamana index, maxDegree=%zu docCnt=%u totalVecs=%u",
max_degree(), doc_cnt(), total_vecs);
if (doc_cnt() != total_vecs) {
LOG_WARN("Index closed abnormally, using totalVecs as curDocCnt");
*mutable_doc_cnt() = total_vecs;
}
// Rebuild key map
if (use_key_info_map_) {
for (node_id_t i = 0; i < doc_cnt(); ++i) {
key_t k = get_key(i);
if (k != kInvalidKey) {
(*keys_map_)[k] = i;
}
}
}
stats_.set_loaded_count(doc_cnt());
return 0;
}
int VamanaStreamerEntity::init_chunks(const Chunk::Pointer &header_chunk) {
// Read header from chunk
const void *hd_ptr = nullptr;
size_t ret = header_chunk->read(0UL, &hd_ptr, header_size());
if (ailego_unlikely(ret != header_size())) {
LOG_ERROR("Read header chunk failed");
return IndexError_ReadData;
}
auto *hd = reinterpret_cast<const VamanaHeader *>(hd_ptr);
// Validate
if (vector_size() != hd->vector_size()) {
LOG_ERROR("vector size %zu mismatch index previous %zu", vector_size(),
hd->vector_size());
return IndexError_Mismatch;
}
if (max_degree() != hd->max_degree()) {
LOG_ERROR("max_degree %zu mismatch index previous %zu", max_degree(),
hd->max_degree());
return IndexError_Mismatch;
}
*mutable_header() = *hd;
// Load node chunks
size_t chunk_cnt = broker_->get_chunk_cnt(ChunkBroker::CHUNK_TYPE_NODE);
for (size_t i = 0; i < chunk_cnt; ++i) {
auto chunk = broker_->get_chunk(ChunkBroker::CHUNK_TYPE_NODE, i);
if (ailego_unlikely(!chunk)) {
LOG_ERROR("Get node chunk %zu failed", i);
return IndexError_ReadData;
}
node_chunks_.emplace_back(std::move(chunk));
}
return 0;
}
int VamanaStreamerEntity::close() {
LOG_DEBUG("close Vamana index");
std::lock_guard<std::mutex> lock(mutex_);
flush_header();
mutable_header()->reset();
keys_map_->clear();
header_.clear();
node_chunks_.clear();
dist_chunks_.clear();
dist_loaded_ = false;
return broker_->close();
}
int VamanaStreamerEntity::flush(uint64_t checkpoint) {
LOG_INFO("Flush Vamana index, curDocs=%u", doc_cnt());
std::lock_guard<std::mutex> lock(mutex_);
flush_header();
return broker_->flush(checkpoint);
}
int VamanaStreamerEntity::dump(const IndexDumper::Pointer &dumper) {
LOG_INFO("Dump Vamana index, curDocs=%u", doc_cnt());
std::vector<key_t> keys(doc_cnt());
auto ret = dump_segments(dumper, keys.data());
if (ailego_unlikely(ret < 0)) {
return static_cast<int>(ret);
}
*stats_.mutable_dumped_size() += ret;
return 0;
}
const VamanaEntity::Pointer VamanaStreamerEntity::clone() const {
std::vector<Chunk::Pointer> cloned_chunks;
cloned_chunks.reserve(node_chunks_.size());
for (size_t i = 0; i < node_chunks_.size(); ++i) {
cloned_chunks.emplace_back(node_chunks_[i]->clone());
if (ailego_unlikely(!cloned_chunks[i])) {
LOG_ERROR("VamanaStreamerEntity get chunk failed in clone");
return VamanaEntity::Pointer();
}
}
auto *entity = new (std::nothrow) VamanaStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_, get_vector_enabled_,
use_key_info_map_, keys_map_lock_, keys_map_, std::move(cloned_chunks),
broker_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("VamanaStreamerEntity new failed");
}
return VamanaEntity::Pointer(entity);
}
const VamanaEntity::Pointer VamanaMmapStreamerEntity::clone() const {
std::vector<Chunk::Pointer> cloned_chunks;
cloned_chunks.reserve(node_chunks_.size());
for (size_t i = 0; i < node_chunks_.size(); ++i) {
cloned_chunks.emplace_back(node_chunks_[i]->clone());
if (ailego_unlikely(!cloned_chunks[i])) {
LOG_ERROR("VamanaMmapStreamerEntity get chunk failed in clone");
return VamanaEntity::Pointer();
}
}
auto *entity = new (std::nothrow) VamanaMmapStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_, get_vector_enabled_,
use_key_info_map_, keys_map_lock_, keys_map_, std::move(cloned_chunks),
broker_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("VamanaMmapStreamerEntity new failed");
}
return VamanaEntity::Pointer(entity);
}
const VamanaEntity::Pointer VamanaContiguousStreamerEntity::clone() const {
std::vector<Chunk::Pointer> cloned_chunks;
cloned_chunks.reserve(node_chunks_.size());
for (size_t i = 0; i < node_chunks_.size(); ++i) {
cloned_chunks.emplace_back(node_chunks_[i]->clone());
if (ailego_unlikely(!cloned_chunks[i])) {
LOG_ERROR("VamanaContiguousStreamerEntity get chunk failed in clone");
return VamanaEntity::Pointer();
}
}
auto *entity = new (std::nothrow) VamanaContiguousStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_, get_vector_enabled_,
use_key_info_map_, keys_map_lock_, keys_map_, std::move(cloned_chunks),
broker_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("VamanaContiguousStreamerEntity new failed");
return VamanaEntity::Pointer();
}
// Share contiguous memory with the clone (zero-copy)
entity->node_memory_ = node_memory_;
entity->node_base_ = node_base_;
return VamanaEntity::Pointer(entity);
}
// ============================================================================
// VamanaContiguousStreamerEntity implementation
// ============================================================================
char *VamanaContiguousStreamerEntity::allocate_contiguous(size_t size) {
if (size == 0) return nullptr;
#if defined(__linux__)
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (ptr == MAP_FAILED) {
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
return nullptr;
}
::madvise(ptr, size, MADV_HUGEPAGE);
return static_cast<char *>(ptr);
#elif defined(__APPLE__)
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANON, -1, 0);
if (ptr == MAP_FAILED) {
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
return nullptr;
}
return static_cast<char *>(ptr);
#elif defined(_WIN32)
void *ptr = ::_aligned_malloc(size, ailego::MemoryHelper::PageSize());
if (!ptr) {
LOG_ERROR("_aligned_malloc failed for contiguous memory, size=%zu", size);
return nullptr;
}
return static_cast<char *>(ptr);
#else
void *ptr = std::aligned_alloc(ailego::MemoryHelper::PageSize(), size);
if (!ptr) {
LOG_ERROR("aligned_alloc failed, size=%zu", size);
return nullptr;
}
return static_cast<char *>(ptr);
#endif
}
int VamanaContiguousStreamerEntity::build_contiguous_memory() {
node_memory_.reset();
node_base_ = nullptr;
const uint32_t total_docs = doc_cnt();
if (total_docs == 0) return 0;
const size_t per_node = node_size();
const size_t total_node_data = static_cast<size_t>(total_docs) * per_node;
size_t node_memory_size = AlignHugePageSize(total_node_data);
char *raw_node = allocate_contiguous(node_memory_size);
if (!raw_node) return IndexError_Runtime;
node_memory_.reset(raw_node, ContiguousDeleter{node_memory_size});
node_base_ = raw_node;
// Copy node data from chunks into contiguous memory
const auto &chunks = node_chunks_;
const uint32_t nodes_per_chunk = 1U << node_index_mask_bits_;
for (size_t chunk_idx = 0; chunk_idx < chunks.size(); ++chunk_idx) {
const void *chunk_data = nullptr;
size_t data_size = chunks[chunk_idx]->data_size();
chunks[chunk_idx]->read(0, &chunk_data, data_size);
uint32_t base_id = chunk_idx * nodes_per_chunk;
uint32_t count_in_chunk = std::min(nodes_per_chunk, total_docs - base_id);
const char *src = static_cast<const char *>(chunk_data);
char *dst = node_base_ + static_cast<size_t>(base_id) * per_node;
std::memcpy(dst, src, static_cast<size_t>(count_in_chunk) * per_node);
}
LOG_INFO(
"Built Vamana contiguous memory: node_size=%zu total_docs=%u "
"node_chunks=%zu",
node_memory_size, total_docs, chunks.size());
return 0;
}
// ============================================================================
// Neighbor distance storage implementation (CSR-like, lazy-loaded)
// ============================================================================
int VamanaStreamerEntity::ensure_dist_storage() {
if (dist_loaded_) return 0;
std::lock_guard<std::mutex> lock(mutex_);
if (dist_loaded_) return 0; // double-check after lock
dist_entry_size_ = static_cast<uint32_t>(max_degree() * sizeof(dist_t));
// Calculate how many dist chunks we need for existing nodes
uint32_t total_docs = doc_cnt();
if (total_docs == 0) {
dist_loaded_ = true;
return 0;
}
// Check if dist chunks already exist in storage (reopened index)
size_t existing_dist_chunks =
broker_->get_chunk_cnt(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST);
if (existing_dist_chunks > 0) {
// Load existing dist chunks
for (size_t i = 0; i < existing_dist_chunks; ++i) {
auto chunk = broker_->get_chunk(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST, i);
if (ailego_unlikely(!chunk)) {
LOG_ERROR("Failed to load dist chunk %zu", i);
return IndexError_ReadData;
}
dist_chunks_.emplace_back(std::move(chunk));
}
LOG_INFO("Loaded %zu existing dist chunks", existing_dist_chunks);
} else {
// Allocate new dist chunks for all existing nodes
int ret = alloc_dist_chunks_for_existing_nodes();
if (ret != 0) return ret;
}
dist_loaded_ = true;
return 0;
}
int VamanaStreamerEntity::ensure_dist_chunk_for(uint32_t chunk_index) {
// No-op when dist storage is not active.
if (!dist_loaded_ || dist_entry_size_ == 0) return 0;
// Idempotent: nothing to do if this dist chunk slot already exists and is
// populated. (Slots created by the placeholder loop below will hold
// nullptr and must still be (re-)allocated.)
if (chunk_index < dist_chunks_.size() && dist_chunks_[chunk_index]) {
return 0;
}
uint32_t dist_chunk_data_size = node_cnt_per_chunk_ * dist_entry_size_;
uint32_t dist_chunk_size = AlignPageSize(dist_chunk_data_size);
auto dp = broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST,
chunk_index, dist_chunk_size);
if (ailego_unlikely(dp.first != 0)) {
LOG_ERROR("Alloc dist chunk %u failed", chunk_index);
return dp.first;
}
dp.second->resize(dist_chunk_data_size);
while (dist_chunks_.size() <= chunk_index) {
dist_chunks_.emplace_back(nullptr);
}
dist_chunks_[chunk_index] = std::move(dp.second);
return 0;
}
int VamanaStreamerEntity::alloc_dist_chunks_for_existing_nodes() {
uint32_t total_docs = doc_cnt();
if (total_docs == 0) return 0;
// Calculate dist chunk size: same number of nodes per chunk as node chunks
uint32_t dist_chunk_data_size = node_cnt_per_chunk_ * dist_entry_size_;
uint32_t dist_chunk_size = AlignPageSize(dist_chunk_data_size);
uint32_t num_chunks_needed =
(total_docs + node_cnt_per_chunk_ - 1) >> node_index_mask_bits_;
for (uint32_t i = 0; i < num_chunks_needed; ++i) {
auto p = broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST, i,
dist_chunk_size);
if (ailego_unlikely(p.first != 0)) {
LOG_ERROR("Alloc dist chunk %u failed", i);
return p.first;
}
// Resize to cover all nodes in this chunk
uint32_t nodes_in_chunk =
std::min(node_cnt_per_chunk_, total_docs - i * node_cnt_per_chunk_);
size_t data_size = static_cast<size_t>(nodes_in_chunk) * dist_entry_size_;
p.second->resize(data_size);
dist_chunks_.emplace_back(std::move(p.second));
}
broker_->mark_dirty();
LOG_INFO("Allocated %u dist chunks for %u existing nodes", num_chunks_needed,
total_docs);
return 0;
}
const dist_t *VamanaStreamerEntity::get_neighbor_dists(node_id_t id) const {
if (!dist_loaded_) return nullptr;
auto loc = get_dist_chunk_loc(id);
if (ailego_unlikely(loc.first >= dist_chunks_.size())) {
sync_dist_chunks(loc.first);
}
if (ailego_unlikely(loc.first >= dist_chunks_.size())) return nullptr;
const void *ptr = nullptr;
size_t ret =
dist_chunks_[loc.first]->read(loc.second, &ptr, dist_entry_size_);
if (ailego_unlikely(ret != dist_entry_size_)) return nullptr;
return static_cast<const dist_t *>(ptr);
}
void VamanaStreamerEntity::update_neighbor_dists(
node_id_t id, const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
if (!dist_loaded_) return;
auto loc = get_dist_chunk_loc(id);
// Dist chunk must have been pre-allocated by add_vector or
// ensure_dist_storage
if (ailego_unlikely(loc.first >= dist_chunks_.size() ||
dist_chunks_[loc.first] == nullptr)) {
LOG_ERROR("Dist chunk %u not allocated for node %u", loc.first, id);
return;
}
// Write distances: fill max_degree slots, zero-pad unused slots
uint32_t max_deg = static_cast<uint32_t>(max_degree());
std::vector<dist_t> dists(max_deg, 0.0f);
for (size_t i = 0; i < neighbors.size() && i < max_deg; ++i) {
dists[i] = neighbors[i].second;
}
dist_chunks_[loc.first]->write(loc.second, dists.data(), dist_entry_size_);
}
void VamanaStreamerEntity::set_neighbor_dist(node_id_t id, uint32_t idx,
dist_t dist) {
if (!dist_loaded_) return;
auto loc = get_dist_chunk_loc(id);
if (ailego_unlikely(loc.first >= dist_chunks_.size())) return;
uint32_t offset = loc.second + idx * sizeof(dist_t);
dist_chunks_[loc.first]->write(offset, &dist, sizeof(dist_t));
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,635 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <iostream>
#include <memory>
#include <mutex>
#if defined(__linux__) || defined(__APPLE__)
#include <sys/mman.h>
#endif
#include <ailego/parallel/lock.h>
#include <sparsehash/dense_hash_map>
#include <zvec/ailego/container/heap.h>
#include <zvec/core/framework/index_framework.h>
#include "algorithm/hnsw/hnsw_chunk.h"
#include "algorithm/hnsw/hnsw_entity.h" // MmapMemoryBlock, BufferPoolMemoryBlock, NeighborsT
#include "vamana_entity.h"
#include "vamana_params.h"
namespace zvec {
namespace core {
// Storage mode for VamanaStreamerEntity
enum class VamanaStorageMode { kMmap = 0, kBufferPool = 1, kContiguous = 2 };
// VamanaStreamerEntity manages vector data, primary keys, and neighbors
// for a single-layer Vamana graph in streaming (incremental) mode.
// Unlike HNSW, Vamana has no upper-level neighbors — only a single
// neighbor list per node. Node layout in chunk:
// [vector_data (vector_size) | key (sizeof(key_t)) | NeighborsHeader +
// neighbors (neighbors_size)]
class VamanaStreamerEntity : public VamanaEntity {
public:
// Virtual interface implementation
virtual int cleanup() override;
virtual const VamanaEntity::Pointer clone() const override;
virtual key_t get_key(node_id_t id) const override;
virtual const void *get_vector(node_id_t id) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
virtual const Neighbors get_neighbors(node_id_t id) const override;
virtual int add_vector(key_t key, const void *vec, node_id_t *id) override;
virtual int add_vector_with_id(node_id_t id, const void *vec) override;
virtual int update_neighbors(
node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
virtual void add_neighbor(node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
virtual int dump(const IndexDumper::Pointer &dumper) override;
virtual void update_entry_point(node_id_t ep) override;
// --- Neighbor distance storage ---
int ensure_dist_storage() override;
bool dist_storage_loaded() const override {
return dist_loaded_;
}
const dist_t *get_neighbor_dists(node_id_t id) const override;
void update_neighbor_dists(
node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
void set_neighbor_dist(node_id_t id, uint32_t idx, dist_t dist) override;
virtual VamanaStorageMode storage_mode() const {
return VamanaStorageMode::kMmap;
}
void set_use_key_info_map(bool use_id_map) {
use_key_info_map_ = use_id_map;
}
public:
VamanaStreamerEntity(IndexStreamer::Stats &stats);
~VamanaStreamerEntity();
virtual const void *get_vector_by_key(key_t key) const override {
auto id = get_id(key);
return id == kInvalidNodeId ? nullptr : get_vector(id);
}
virtual int get_vector_by_key(
const key_t key, IndexStorage::MemoryBlock &block) const override {
auto id = get_id(key);
if (id != kInvalidNodeId) {
return get_vector(id, block);
}
return IndexError_InvalidArgument;
}
int init(size_t max_doc_cnt);
int flush(uint64_t checkpoint);
int open(IndexStorage::Pointer stg, uint64_t max_index_size, bool check_crc);
int close();
int set_index_meta(const IndexMeta &meta) const {
return IndexHelper::SerializeToStorage(meta, broker_->storage().get());
}
int get_index_meta(IndexMeta *meta) const {
return IndexHelper::DeserializeFromStorage(broker_->storage().get(), meta);
}
inline void set_chunk_size(size_t val) {
chunk_size_ = val;
}
inline void set_get_vector(bool val) {
get_vector_enabled_ = val;
}
inline node_id_t get_id(key_t key) const {
if (use_key_info_map_) {
keys_map_lock_->lock_shared();
auto it = keys_map_->find(key);
keys_map_lock_->unlock_shared();
return it == keys_map_->end() ? kInvalidNodeId : it->second;
}
return key;
}
// --- Typed access methods for hot-path optimization ---
// These are templated on MemBlock type to avoid runtime branching.
template <typename MemBlock>
inline NeighborsT<MemBlock> get_neighbors_typed(node_id_t id) const;
template <typename MemBlock>
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MemBlock> &vec_blocks) const;
template <typename MemBlock>
inline key_t get_key_typed(node_id_t id) const;
protected:
inline void sync_node_chunks(size_t idx) const {
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, idx, &node_chunks_);
}
protected:
template <class Key, class T>
using HashMap = google::dense_hash_map<Key, T, std::hash<Key>>;
template <class Key, class T>
using HashMapPointer = std::shared_ptr<HashMap<Key, T>>;
//! Clone constructor, used by clone method in subclasses
VamanaStreamerEntity(IndexStreamer::Stats &stats, const VamanaHeader &hd,
size_t chunk_size, uint32_t node_index_mask_bits,
bool get_vector_enabled, bool use_key_info_map,
std::shared_ptr<ailego::SharedMutex> &keys_map_lock,
const HashMapPointer<key_t, node_id_t> &keys_map,
std::vector<Chunk::Pointer> &&node_chunks,
const ChunkBroker::Pointer &broker)
: stats_(stats),
chunk_size_(chunk_size),
node_index_mask_bits_(node_index_mask_bits),
node_cnt_per_chunk_(1UL << node_index_mask_bits_),
node_index_mask_(node_cnt_per_chunk_ - 1),
get_vector_enabled_(get_vector_enabled),
use_key_info_map_(use_key_info_map),
keys_map_lock_(keys_map_lock),
keys_map_(keys_map),
node_chunks_(std::move(node_chunks)),
broker_(broker) {
*mutable_header() = hd;
neighbor_size_ = neighbors_size();
}
//! Lazy chunk synchronization: fetches chunks from broker when needed.
//! Each clone entity has its own node_chunks_ vector, so concurrent
//! search threads do not race with the writer's emplace_back.
void sync_chunks(ChunkBroker::CHUNK_TYPE type, size_t idx,
std::vector<Chunk::Pointer> *chunks) const {
if (ailego_likely(idx < chunks->size())) {
return;
}
for (size_t i = chunks->size(); i <= idx; ++i) {
auto chunk = broker_->get_chunk(type, i);
ailego_assert_with(!!chunk, "get chunk failed");
chunks->emplace_back(std::move(chunk));
}
}
inline std::pair<uint32_t, uint32_t> get_vector_chunk_loc(
node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size();
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
return std::make_pair(chunk_idx, offset);
}
inline std::pair<uint32_t, uint32_t> get_key_chunk_loc(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
return std::make_pair(chunk_idx, offset);
}
inline std::pair<Chunk *, size_t> get_neighbor_chunk_loc(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
ailego_assert_abort(chunk_idx < node_chunks_.size(), "invalid chunk idx");
return std::make_pair(node_chunks_[chunk_idx].get(), offset);
}
// Get chunk location for neighbor distance data.
// Uses the same chunk indexing as node chunks but with dist_entry_size_.
inline std::pair<uint32_t, uint32_t> get_dist_chunk_loc(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * dist_entry_size_;
return std::make_pair(chunk_idx, offset);
}
void sync_dist_chunks(size_t idx) const {
sync_chunks(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST, idx, &dist_chunks_);
}
int ensure_dist_chunk_for(uint32_t chunk_index);
int alloc_dist_chunks_for_existing_nodes();
size_t estimate_doc_capacity() const {
return node_chunks_.capacity() * node_cnt_per_chunk_;
}
int init_chunk_params(size_t max_index_size, bool huge_page) {
node_cnt_per_chunk_ = std::max<uint32_t>(1, chunk_size_ / node_size());
node_index_mask_bits_ = std::ceil(std::log2(node_cnt_per_chunk_));
node_cnt_per_chunk_ = 1UL << node_index_mask_bits_;
if (huge_page) {
chunk_size_ = AlignHugePageSize(node_cnt_per_chunk_ * node_size());
} else {
chunk_size_ = AlignPageSize(node_cnt_per_chunk_ * node_size());
}
node_index_mask_ = node_cnt_per_chunk_ - 1;
if (max_index_size == 0UL) {
max_index_size_ = chunk_size_ * kDefaultMaxChunkCnt;
} else {
max_index_size_ = max_index_size;
}
size_t max_node_chunk_cnt =
std::ceil(static_cast<double>(max_index_size_) / chunk_size_);
node_chunks_.reserve(max_node_chunk_cnt);
LOG_DEBUG(
"VamanaSettings: nodeSize=%zu chunkSize=%u nodeCntPerChunk=%u "
"maxChunkCnt=%zu maxIndexSize=%zu",
node_size(), chunk_size_, node_cnt_per_chunk_, max_node_chunk_cnt,
max_index_size_);
return 0;
}
int init_chunks(const Chunk::Pointer &header_chunk);
int flush_header(void) {
if (!broker_->dirty()) {
return 0;
}
auto header_chunk = broker_->get_chunk(ChunkBroker::CHUNK_TYPE_HEADER,
ChunkBroker::kDefaultChunkSeqId);
if (ailego_unlikely(!header_chunk)) {
LOG_ERROR("get header chunk failed");
return IndexError_Runtime;
}
size_t size = header_chunk->write(0UL, &header(), header_size());
if (ailego_unlikely(size != header_size())) {
LOG_ERROR("Write header chunk failed");
return IndexError_WriteData;
}
return 0;
}
protected:
IndexStreamer::Stats &stats_;
std::mutex mutex_{};
size_t max_index_size_{0UL};
uint32_t chunk_size_{kDefaultChunkSize};
uint32_t node_index_mask_bits_{0U};
uint32_t node_cnt_per_chunk_{0U};
uint32_t node_index_mask_{0U};
uint32_t neighbor_size_{0U};
bool get_vector_enabled_{false};
bool use_key_info_map_{true};
mutable std::shared_ptr<ailego::SharedMutex> keys_map_lock_;
HashMapPointer<key_t, node_id_t> keys_map_;
ChunkBroker::Pointer broker_;
mutable std::vector<Chunk::Pointer> node_chunks_{};
private:
mutable std::vector<Chunk::Pointer> dist_chunks_{};
bool dist_loaded_{false};
uint32_t dist_entry_size_{0}; // max_degree * sizeof(dist_t)
};
// --- Template specializations for typed MemoryBlock access ---
template <>
inline NeighborsT<MmapMemoryBlock>
VamanaStreamerEntity::get_neighbors_typed<MmapMemoryBlock>(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
ailego_assert_with(offset < node_chunks_[chunk_idx]->data_size(),
"invalid chunk offset");
const void *ptr = nullptr;
size_t ret = node_chunks_[chunk_idx]->read(offset, &ptr, neighbor_size_);
if (ailego_unlikely(ret != neighbor_size_)) {
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
return NeighborsT<MmapMemoryBlock>();
}
MmapMemoryBlock block(const_cast<void *>(ptr));
return NeighborsT<MmapMemoryBlock>(std::move(block));
}
template <>
inline NeighborsT<BufferPoolMemoryBlock>
VamanaStreamerEntity::get_neighbors_typed<BufferPoolMemoryBlock>(
node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
IndexStorage::MemoryBlock mem_block;
size_t ret = node_chunks_[chunk_idx]->read(offset, mem_block, neighbor_size_);
if (ailego_unlikely(ret != neighbor_size_)) {
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
return NeighborsT<BufferPoolMemoryBlock>();
}
BufferPoolMemoryBlock block(mem_block.buffer_pool_handle_,
mem_block.buffer_block_id_, mem_block.data_);
mem_block.buffer_pool_handle_ = nullptr;
return NeighborsT<BufferPoolMemoryBlock>(std::move(block));
}
template <>
inline int VamanaStreamerEntity::get_vector_typed<MmapMemoryBlock>(
const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
auto loc = get_vector_chunk_loc(ids[i]);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
const void *ptr = nullptr;
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, vector_size());
if (ailego_unlikely(ret != vector_size())) {
LOG_ERROR("Read vector failed, ret=%zu", ret);
return IndexError_ReadData;
}
vec_blocks[i].reset(const_cast<void *>(ptr));
}
return 0;
}
template <>
inline int VamanaStreamerEntity::get_vector_typed<BufferPoolMemoryBlock>(
const node_id_t *ids, uint32_t count,
std::vector<BufferPoolMemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
auto loc = get_vector_chunk_loc(ids[i]);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
IndexStorage::MemoryBlock mem_block;
size_t ret =
node_chunks_[loc.first]->read(loc.second, mem_block, vector_size());
if (ailego_unlikely(ret != vector_size())) {
LOG_ERROR("Read vector failed, ret=%zu", ret);
return IndexError_ReadData;
}
vec_blocks[i] =
BufferPoolMemoryBlock(mem_block.buffer_pool_handle_,
mem_block.buffer_block_id_, mem_block.data_);
mem_block.buffer_pool_handle_ = nullptr;
}
return 0;
}
template <>
inline key_t VamanaStreamerEntity::get_key_typed<MmapMemoryBlock>(
node_id_t id) const {
if (!use_key_info_map_) return id;
auto loc = get_key_chunk_loc(id);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
const void *ptr = nullptr;
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, sizeof(key_t));
if (ailego_unlikely(ret != sizeof(key_t))) {
LOG_ERROR("Read key failed, ret=%zu", ret);
return kInvalidKey;
}
return *reinterpret_cast<const key_t *>(ptr);
}
template <>
inline key_t VamanaStreamerEntity::get_key_typed<BufferPoolMemoryBlock>(
node_id_t id) const {
if (!use_key_info_map_) return id;
auto loc = get_key_chunk_loc(id);
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
IndexStorage::MemoryBlock key_block;
size_t ret =
node_chunks_[loc.first]->read(loc.second, key_block, sizeof(key_t));
if (ailego_unlikely(ret != sizeof(key_t))) {
LOG_ERROR("Read key failed, ret=%zu", ret);
return kInvalidKey;
}
return *reinterpret_cast<const key_t *>(key_block.data());
}
// --- Typed entity subclass for mmap mode ---
// Caches chunk base addresses to eliminate virtual function calls on the
// search hot path. For mmap mode, chunk data is memory-mapped at init time,
// so we can directly compute pointers via base_addr + offset.
class VamanaMmapStreamerEntity : public VamanaStreamerEntity {
public:
using MemoryBlock = MmapMemoryBlock;
using TypedNeighbors = NeighborsT<MmapMemoryBlock>;
using VamanaStreamerEntity::VamanaStreamerEntity;
VamanaStorageMode storage_mode() const override {
return VamanaStorageMode::kMmap;
}
//! Override clone to return correct subclass type, so that
//! static_cast<const VamanaMmapStreamerEntity&> in the algorithm is safe.
const VamanaEntity::Pointer clone() const override;
inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
const char *base = get_node_chunk_base(chunk_idx);
MmapMemoryBlock block(const_cast<char *>(base + offset));
return TypedNeighbors(std::move(block));
}
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
uint32_t chunk_idx = ids[i] >> node_index_mask_bits_;
uint32_t offset = (ids[i] & node_index_mask_) * node_size();
const char *base = get_node_chunk_base(chunk_idx);
vec_blocks[i].reset(const_cast<char *>(base + offset));
}
return 0;
}
inline key_t get_key_typed(node_id_t id) const {
if (!use_key_info_map_) return id;
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
const char *base = get_node_chunk_base(chunk_idx);
return *reinterpret_cast<const key_t *>(base + offset);
}
private:
inline const char *get_node_chunk_base(uint32_t chunk_idx) const {
if (ailego_unlikely(chunk_idx >= node_chunk_bases_.size())) {
sync_node_chunk_bases(chunk_idx);
}
return node_chunk_bases_[chunk_idx];
}
void sync_node_chunk_bases(uint32_t chunk_idx) const {
sync_node_chunks(chunk_idx);
const auto &chunks = node_chunks_;
for (size_t i = node_chunk_bases_.size(); i <= chunk_idx; ++i) {
const void *ptr = nullptr;
chunks[i]->read(0, &ptr, 1);
node_chunk_bases_.push_back(static_cast<const char *>(ptr));
}
}
mutable std::vector<const char *> node_chunk_bases_{};
};
// --- Typed entity subclass for buffer pool mode ---
class VamanaBufferPoolStreamerEntity : public VamanaStreamerEntity {
public:
using MemoryBlock = BufferPoolMemoryBlock;
using TypedNeighbors = NeighborsT<BufferPoolMemoryBlock>;
using VamanaStreamerEntity::VamanaStreamerEntity;
VamanaStorageMode storage_mode() const override {
return VamanaStorageMode::kBufferPool;
}
inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
return VamanaStreamerEntity::get_neighbors_typed<BufferPoolMemoryBlock>(id);
}
inline int get_vector_typed(
const node_id_t *ids, uint32_t count,
std::vector<BufferPoolMemoryBlock> &vec_blocks) const {
return VamanaStreamerEntity::get_vector_typed<BufferPoolMemoryBlock>(
ids, count, vec_blocks);
}
inline key_t get_key_typed(node_id_t id) const {
return VamanaStreamerEntity::get_key_typed<BufferPoolMemoryBlock>(id);
}
};
// --- Typed entity subclass for contiguous memory mode ---
// Allocates contiguous memory and copies all chunk data into it.
// Access is via a single base pointer + offset, eliminating chunk-level
// indirection and maximizing memory locality.
class VamanaContiguousStreamerEntity : public VamanaMmapStreamerEntity {
public:
using VamanaMmapStreamerEntity::VamanaMmapStreamerEntity;
VamanaStorageMode storage_mode() const override {
return VamanaStorageMode::kContiguous;
}
//! Override clone to return correct subclass type.
//! Cloned entity shares contiguous memory via shared_ptr.
const VamanaEntity::Pointer clone() const override;
~VamanaContiguousStreamerEntity() = default;
// Build contiguous memory from chunks after open.
int build_contiguous_memory();
//! Degrade to mmap mode by releasing contiguous memory and falling back
//! to chunk-based access.
void degrade_to_mmap() {
node_memory_.reset();
node_base_ = nullptr;
LOG_INFO("Vamana contiguous entity degraded to mmap mode for insertion");
}
bool is_contiguous() const {
return node_base_ != nullptr;
}
int add_vector(key_t key, const void *vec, node_id_t *id) override {
if (ailego_unlikely(is_contiguous())) degrade_to_mmap();
return VamanaMmapStreamerEntity::add_vector(key, vec, id);
}
int add_vector_with_id(node_id_t id, const void *vec) override {
if (ailego_unlikely(is_contiguous())) degrade_to_mmap();
return VamanaMmapStreamerEntity::add_vector_with_id(id, vec);
}
inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
if (ailego_likely(node_base_ != nullptr)) {
const char *ptr = node_base_ + static_cast<size_t>(id) * node_size() +
vector_size() + sizeof(key_t);
MmapMemoryBlock block(const_cast<char *>(ptr));
return TypedNeighbors(std::move(block));
}
return VamanaMmapStreamerEntity::get_neighbors_typed(id);
}
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
if (ailego_likely(node_base_ != nullptr)) {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
const char *ptr =
node_base_ + static_cast<size_t>(ids[i]) * node_size();
vec_blocks[i].reset(const_cast<char *>(ptr));
}
return 0;
}
return VamanaMmapStreamerEntity::get_vector_typed(ids, count, vec_blocks);
}
inline key_t get_key_typed(node_id_t id) const {
if (ailego_likely(node_base_ != nullptr)) {
if (!use_key_info_map_) return id;
const char *ptr =
node_base_ + static_cast<size_t>(id) * node_size() + vector_size();
return *reinterpret_cast<const key_t *>(ptr);
}
return VamanaMmapStreamerEntity::get_key_typed(id);
}
protected:
//! Custom deleter for contiguous memory (munmap / _aligned_free / free)
struct ContiguousDeleter {
size_t size;
void operator()(char *ptr) const {
if (!ptr) return;
#if defined(__linux__) || defined(__APPLE__)
::munmap(ptr, size);
#elif defined(_WIN32)
::_aligned_free(ptr);
#else
std::free(ptr);
#endif
}
};
//! Shared ownership of contiguous memory (enables zero-copy clone)
std::shared_ptr<char> node_memory_{};
//! Raw pointer for hot-path access (derived from shared_ptr)
char *node_base_{nullptr};
private:
static char *allocate_contiguous(size_t size);
};
} // namespace core
} // namespace zvec