feat(entity/search): add LinearPool/BlockHeap, refac entity layout and access. (#450)

* feat(entity/search): add LinearPool/BlockHeap, split contiguous entity layout and add direct-pointer fast search path

Introduce single-heap candidate structures (LinearPool/BlockHeap) and
refactor greedy search to dispatch between pool and dual-heap paths.
Split node layout into separate vector/graph arrays for better cache
locality, add zero-copy get_vector_ptr() on the hot path, and extract
huge-page allocation into MemoryHelper. Also adds pyglass attribution.
This commit is contained in:
luoxiaojian
2026-06-08 13:51:46 +08:00
committed by GitHub
parent af88794463
commit dd4d8117df
24 changed files with 1504 additions and 288 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ cc_library(
NAME core_knn_hnsw
STATIC SHARED STRICT ALWAYS_LINK
SRCS *.cc
LIBS core_framework sparsehash
LIBS core_framework core_utility sparsehash
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
LDFLAGS "${CORE_KNN_HNSW_LDFLAGS}"
VERSION "${PROXIMA_ZVEC_VERSION}"
+194 -30
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_algorithm.h"
#include <type_traits>
namespace zvec {
namespace core {
@@ -48,7 +49,7 @@ int HnswAlgorithm<EntityType>::add_node(node_id_t id, level_t level,
for (; cur_level >= 0; --cur_level) {
search_neighbors(cur_level, &entry_point, &dist, ctx->level_topk(cur_level),
ctx);
ctx, /*use_pool=*/false);
}
// add neighbors from down level to top level, to avoid upper level visible
@@ -86,7 +87,7 @@ int HnswAlgorithm<EntityType>::search(HnswContext *ctx) const {
auto &topk_heap = ctx->topk_heap();
topk_heap.clear();
search_neighbors(0, &entry_point, &dist, topk_heap, ctx);
search_neighbors(0, &entry_point, &dist, topk_heap, ctx, /*use_pool=*/true);
if (ctx->group_by_search()) {
expand_neighbors_by_group(topk_heap, ctx);
@@ -170,21 +171,117 @@ void HnswAlgorithm<EntityType>::add_neighbors(node_id_t id, level_t level,
return;
}
template <typename EntityType>
void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
node_id_t *entry_point,
dist_t *dist, TopkHeap &topk,
HnswContext *ctx) const {
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
HnswDistCalculator &dc = ctx->dist_calculator();
// ============================================================================
// search_neighbors helper templates
//
// Two specialized inner loops, dispatched from search_neighbors():
//
// fast_search_neighbors: mmap/contiguous with direct vector pointers.
// Uses BlockHeap (AVX2) or LinearPool (scalar)
// for visited tracking and top-k maintenance.
// dual_heap_search_neighbors: CandidateHeap + TopkHeap + VisitFilter.
// Used for add_node (use_pool=false), filtered
// search, upper levels, and BufferPool fallback.
// ============================================================================
// mmap/contiguous variant: resolve vectors via get_vector_ptr and use
// LinearPool or BlockHeap for visited tracking + top-k maintenance.
// HeapType must expose reset/set_visited/check_visited/push_block/has_next/pop.
template <typename EntityType, typename HeapType>
void fast_search_neighbors(const EntityType &entity, HeapType &pool,
VisitFilter &visit, HnswDistCalculator &dc,
uint32_t topk, uint32_t ef, node_id_t entry_point,
dist_t entry_dist, uint32_t prefetch_lines) {
const uint32_t max_deg = entity.max_degree(0); // level 0 only
const uint32_t cap = std::max(topk, ef);
pool.reset(static_cast<int32_t>(cap), static_cast<int32_t>(max_deg));
visit.clear();
visit.set_visited(entry_point);
pool.push_block(&entry_dist, &entry_point, 1);
static constexpr uint32_t GRAPH_PO = 8;
uint32_t buf_capacity = max_deg;
std::vector<node_id_t> neighbor_ids(buf_capacity);
std::vector<float> dists(buf_capacity);
std::vector<const void *> neighbor_vecs(buf_capacity);
while (pool.has_next()) {
auto current_node = pool.pop();
const auto neighbors = entity.get_neighbors_typed(0, current_node);
ailego_prefetch(neighbors.data);
if (neighbors.size() > buf_capacity) {
buf_capacity = neighbors.size();
neighbor_ids.resize(buf_capacity);
dists.resize(buf_capacity);
neighbor_vecs.resize(buf_capacity);
}
const uint32_t po =
std::min(static_cast<uint32_t>(neighbors.size()), GRAPH_PO);
uint32_t unvisited_count = 0;
uint32_t i = 0;
// Phase 1: scan first `po` neighbors with prefetch.
for (; i < po; ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) continue;
visit.set_visited(node);
const void *vec_ptr = entity.get_vector_ptr(node);
const char *p = reinterpret_cast<const char *>(vec_ptr);
for (uint32_t cl = 0; cl < prefetch_lines; ++cl) {
ailego_prefetch(p + cl * 64);
}
neighbor_ids[unvisited_count] = node;
neighbor_vecs[unvisited_count] = vec_ptr;
unvisited_count++;
}
// Phase 2: scan remaining neighbors.
for (; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) continue;
visit.set_visited(node);
neighbor_ids[unvisited_count] = node;
neighbor_vecs[unvisited_count] = entity.get_vector_ptr(node);
unvisited_count++;
}
if (unvisited_count == 0) continue;
dc.batch_dist(neighbor_vecs.data(), unvisited_count, dists.data());
pool.push_block(dists.data(), neighbor_ids.data(),
static_cast<int32_t>(unvisited_count));
}
}
// ============================================================================
// dual_heap_search_neighbors: shared core for the fallback dual-heap path.
//
// Maintains a candidate min-heap + topk heap + VisitFilter. Supports
// arbitrary levels, filters, and MemoryBlock types (BufferPool/Mmap).
// Also updates entry_point/dist for next-level continuation.
// ============================================================================
template <typename EntityType, typename MemBlockType, typename FilterFn>
void dual_heap_search_neighbors(const EntityType &entity, level_t level,
node_id_t *entry_point, dist_t *dist,
TopkHeap &topk, HnswContext *ctx,
HnswDistCalculator &dc, FilterFn &&filter) {
static constexpr uint32_t BATCH_SIZE = 12;
static constexpr uint32_t PREFETCH_STEP = 2;
uint32_t buf_capacity = entity.max_degree(level);
std::vector<node_id_t> neighbor_ids(buf_capacity);
std::vector<MemBlockType> neighbor_vec_blocks;
neighbor_vec_blocks.reserve(buf_capacity);
std::vector<float> dists(buf_capacity);
std::vector<const void *> neighbor_vecs(buf_capacity);
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (ctx->filter().is_valid()) {
filter = [&](node_id_t id) {
return ctx->filter()(entity.get_key_typed(id));
};
}
candidates.clear();
visit.clear();
@@ -210,7 +307,14 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
(*ctx->mutable_stats_get_neighbors())++;
}
std::vector<node_id_t> neighbor_ids(neighbors.size());
if (neighbors.size() > buf_capacity) {
buf_capacity = neighbors.size();
neighbor_ids.resize(buf_capacity);
neighbor_vec_blocks.resize(buf_capacity);
dists.resize(buf_capacity);
neighbor_vecs.resize(buf_capacity);
}
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
@@ -227,7 +331,7 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
continue;
}
std::vector<MemBlockType> neighbor_vec_blocks;
neighbor_vec_blocks.clear();
int ret =
entity.get_vector_typed(neighbor_ids.data(), size, neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
@@ -238,15 +342,9 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
}
// do prefetch
static constexpr node_id_t BATCH_SIZE = 12;
static constexpr node_id_t PREFETCH_STEP = 2;
for (uint32_t i = 0; i < std::min(BATCH_SIZE * PREFETCH_STEP, size); ++i) {
ailego_prefetch(neighbor_vec_blocks[i].data());
}
// done
std::vector<float> dists(size);
std::vector<const void *> neighbor_vecs(size);
for (uint32_t i = 0; i < size; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
@@ -268,11 +366,77 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
if (!filter(node)) {
topk.emplace(node, cur_dist);
}
} // end if
} // end for
} // while
}
}
}
}
return;
// ============================================================================
// search_neighbors: Dispatch to fast or dual-heap path.
//
// - add_node / filtered / upper levels → dual_heap_search_neighbors
// - level-0 unfiltered search:
// MmapMemoryBlock → fast_search_neighbors (BlockHeap/LinearPool)
// BufferPool → dual_heap_search_neighbors (fallback)
// ============================================================================
template <typename EntityType>
void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
node_id_t *entry_point,
dist_t *dist, TopkHeap &topk,
HnswContext *ctx,
bool use_pool) const {
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
HnswDistCalculator &dc = ctx->dist_calculator();
const uint32_t prefetch_lines = (entity.vector_size() + 63) / 64;
if (!use_pool || ctx->filter().is_valid() || level != 0) {
// Dual-heap path: add_node, filtered search, or upper-level scan.
auto run_with_filter = [&](auto &&filter) {
dual_heap_search_neighbors<EntityType, MemBlockType>(
entity, level, entry_point, dist, topk, ctx, dc,
std::forward<decltype(filter)>(filter));
};
if (ctx->filter().is_valid()) {
auto filter = [&](node_id_t id) {
return ctx->filter()(entity.get_key_typed(id));
};
run_with_filter(filter);
} else {
auto filter = [](node_id_t) { return false; };
run_with_filter(filter);
}
} else {
// Pool-based path for level-0 unfiltered search.
if constexpr (std::is_same_v<MemBlockType, MmapMemoryBlock>) {
// Fast path: direct pointer access via get_vector_ptr.
// BlockHeap (AVX2) or LinearPool (scalar) for top-k tracking.
const uint32_t topk_v = static_cast<uint32_t>(ctx->topk());
const uint32_t ef_v = ctx->ef();
const bool avx2_ok =
zvec::ailego::internal::CpuFeatures::static_flags_.AVX2;
auto &visit = ctx->visit_filter();
if (avx2_ok) {
auto &bpool = ctx->block_pool();
fast_search_neighbors(entity, bpool, visit, dc, topk_v, ef_v,
*entry_point, *dist, prefetch_lines);
copy_pool_to_topk(bpool, topk);
} else {
auto &lpool = ctx->pool();
fast_search_neighbors(entity, lpool, visit, dc, topk_v, ef_v,
*entry_point, *dist, prefetch_lines);
copy_pool_to_topk(lpool, topk);
}
} else {
// BufferPool entities: fallback to dual-heap path.
auto filter = [](node_id_t) { return false; };
dual_heap_search_neighbors<EntityType, MemBlockType>(
entity, level, entry_point, dist, topk, ctx, dc, filter);
}
}
}
template <typename EntityType>
@@ -390,9 +554,9 @@ void HnswAlgorithm<EntityType>::expand_neighbors_by_group(
}
candidates.emplace(node, cur_dist);
} // end for
} // end while
} // end if
}
}
}
}
template <typename EntityType>
+7 -4
View File
@@ -111,11 +111,14 @@ class HnswAlgorithm : public HnswAlgorithmBase {
void add_neighbors(node_id_t id, level_t level, TopkHeap &topk_heap,
HnswContext *ctx);
//! Given a node id and level, search the nearest neighbors in graph
//! Note: the nearest neighbors result keeps in topk, and entry_point and
//! dist will be updated to current level nearest node id and distance
//! Given a node id and level, search the nearest neighbors in graph.
//! Dispatches to fast_search_neighbors (pool-based, direct pointer) for
//! mmap/contiguous level-0 unfiltered search, or dual_heap_search_neighbors
//! (CandidateHeap + TopkHeap) for add_node, filtered search, upper levels,
//! and BufferPool fallback.
//! Note: entry_point and dist will be updated to current level nearest node.
void search_neighbors(level_t level, node_id_t *entry_point, dist_t *dist,
TopkHeap &topk, HnswContext *ctx) const;
TopkHeap &topk, HnswContext *ctx, bool use_pool) const;
//! Update the node's neighbors
void update_neighbors(HnswDistCalculator &dc, node_id_t id, level_t level,
+18
View File
@@ -14,6 +14,8 @@
#pragma once
#include <zvec/core/framework/index_context.h>
#include "utility/block_heap.h"
#include "utility/linear_pool.h"
#include "utility/sparse_utility.h"
#include "utility/visit_filter.h"
#include "hnsw_dist_calculator.h"
@@ -275,6 +277,15 @@ class HnswContext : public IndexContext {
return update_heap_;
}
inline LinearPool<dist_t> &pool() {
return pool_;
}
// Only accessed under a runtime CpuFeatures::AVX2 guard at call sites.
inline BlockHeap &block_pool() {
return block_pool_;
}
inline VisitFilter &visit_filter() {
return visit_filter_;
}
@@ -299,6 +310,10 @@ class HnswContext : public IndexContext {
ef_ = v;
}
inline uint32_t ef(void) const {
return ef_;
}
inline void set_filter_mode(uint32_t v) {
filter_mode_ = v;
}
@@ -530,6 +545,9 @@ class HnswContext : public IndexContext {
uint32_t stats_get_vector_cnt_{0u};
uint32_t stats_visit_dup_cnt_{0u};
std::string preprocess_buffer_;
LinearPool<dist_t> pool_;
BlockHeap block_pool_;
};
} // namespace core
+58 -23
View File
@@ -844,8 +844,11 @@ const HnswEntity::Pointer HnswContiguousStreamerEntity::clone() const {
}
// Share contiguous memory with the clone (zero-copy)
entity->node_memory_ = node_memory_;
entity->node_base_ = node_base_;
entity->vector_memory_ = vector_memory_;
entity->vector_base_ = vector_base_;
entity->graph_memory_ = graph_memory_;
entity->graph_base_ = graph_base_;
entity->graph_stride_ = graph_stride_;
entity->upper_neighbor_memory_ = upper_neighbor_memory_;
entity->upper_neighbor_base_ = upper_neighbor_base_;
entity->upper_chunk_offsets_ = upper_chunk_offsets_;
@@ -900,8 +903,10 @@ char *HnswContiguousStreamerEntity::allocate_contiguous(size_t size) {
}
int HnswContiguousStreamerEntity::build_contiguous_memory() {
node_memory_.reset();
node_base_ = nullptr;
vector_memory_.reset();
vector_base_ = nullptr;
graph_memory_.reset();
graph_base_ = nullptr;
upper_neighbor_memory_.reset();
upper_neighbor_base_ = nullptr;
upper_chunk_offsets_.clear();
@@ -911,20 +916,36 @@ int HnswContiguousStreamerEntity::build_contiguous_memory() {
return 0;
}
// --- Build contiguous node memory ---
const size_t per_node = node_size();
const size_t total_node_data = static_cast<size_t>(total_docs) * per_node;
size_t node_memory_size = AlignHugePageSize(total_node_data);
char *raw_node = allocate_contiguous(node_memory_size);
if (!raw_node) {
const size_t vec_size = vector_size();
// graph_stride = key + L0 neighbors (everything except vector)
graph_stride_ = sizeof(key_t) + neighbor_size_;
// --- Allocate flat vector array (stride = vector_size) ---
const size_t total_vec_data = static_cast<size_t>(total_docs) * vec_size;
size_t vector_memory_size = AlignHugePageSize(total_vec_data);
char *raw_vec = allocate_contiguous(vector_memory_size);
if (!raw_vec) {
return IndexError_Runtime;
}
node_memory_.reset(raw_node, ContiguousDeleter{node_memory_size});
node_base_ = raw_node;
vector_memory_.reset(raw_vec, ContiguousDeleter{vector_memory_size});
vector_base_ = raw_vec;
// Copy node data from chunks into contiguous memory
// Each chunk holds node_cnt_per_chunk nodes, laid out at offset
// (id & mask) * node_size within the chunk.
// --- Allocate graph array (stride = sizeof(key_t) + neighbor_size) ---
const size_t total_graph_data =
static_cast<size_t>(total_docs) * graph_stride_;
size_t graph_memory_size = AlignHugePageSize(total_graph_data);
char *raw_graph = allocate_contiguous(graph_memory_size);
if (!raw_graph) {
vector_memory_.reset();
vector_base_ = nullptr;
return IndexError_Runtime;
}
graph_memory_.reset(raw_graph, ContiguousDeleter{graph_memory_size});
graph_base_ = raw_graph;
// Split node data from chunks into vector and graph arrays.
// Original node layout: [vector (vec_size) | key (8B) | L0 neighbors]
const auto &chunks = node_chunks_;
const uint32_t nodes_per_chunk = 1U << node_index_mask_bits_;
for (size_t chunk_idx = 0; chunk_idx < chunks.size(); ++chunk_idx) {
@@ -932,19 +953,30 @@ int HnswContiguousStreamerEntity::build_contiguous_memory() {
size_t data_size = chunks[chunk_idx]->data_size();
chunks[chunk_idx]->read(0, &chunk_data, data_size);
// Number of nodes in this chunk
uint32_t base_id = chunk_idx * nodes_per_chunk;
uint32_t count_in_chunk = std::min(nodes_per_chunk, total_docs - base_id);
// Copy each node's data
const char *src = static_cast<const char *>(chunk_data);
char *dst = node_base_ + static_cast<size_t>(base_id) * per_node;
std::memcpy(dst, src, static_cast<size_t>(count_in_chunk) * per_node);
for (uint32_t i = 0; i < count_in_chunk; ++i) {
const char *node_src = src + static_cast<size_t>(i) * per_node;
size_t global_id = static_cast<size_t>(base_id + i);
// Copy vector to flat vector array
std::memcpy(vector_base_ + global_id * vec_size, node_src, vec_size);
// Copy key + L0 neighbors to graph array
std::memcpy(graph_base_ + global_id * graph_stride_, node_src + vec_size,
graph_stride_);
}
}
// --- Build contiguous upper neighbor memory ---
const auto &upper_chunks = upper_neighbor_chunks_;
if (upper_chunks.empty()) {
LOG_INFO(
"Built HNSW contiguous memory (split layout): "
"vector_mem=%zu graph_mem=%zu total_docs=%u node_chunks=%zu",
vector_memory_size, graph_memory_size, total_docs, chunks.size());
return 0;
}
@@ -962,8 +994,10 @@ int HnswContiguousStreamerEntity::build_contiguous_memory() {
size_t upper_memory_size = AlignHugePageSize(total_upper_size);
char *raw_upper = allocate_contiguous(upper_memory_size);
if (!raw_upper) {
node_memory_.reset();
node_base_ = nullptr;
vector_memory_.reset();
vector_base_ = nullptr;
graph_memory_.reset();
graph_base_ = nullptr;
return IndexError_Runtime;
}
upper_neighbor_memory_.reset(raw_upper, ContiguousDeleter{upper_memory_size});
@@ -979,10 +1013,11 @@ int HnswContiguousStreamerEntity::build_contiguous_memory() {
}
LOG_INFO(
"Built contiguous memory: node_size=%zu upper_neighbor_size=%zu "
"Built HNSW contiguous memory (split layout): "
"vector_mem=%zu graph_mem=%zu upper_neighbor_mem=%zu "
"total_docs=%u node_chunks=%zu upper_chunks=%zu",
node_memory_size, upper_memory_size, total_docs, chunks.size(),
upper_chunks.size());
vector_memory_size, graph_memory_size, upper_memory_size, total_docs,
chunks.size(), upper_chunks.size());
return 0;
}
+70 -30
View File
@@ -217,6 +217,10 @@ class HnswStreamerEntity : public HnswEntity {
return sizeof(NeighborsHeader) + upper_neighbor_cnt() * sizeof(node_id_t);
}
inline size_t max_degree(level_t level) const {
return level == 0 ? neighbor_size_ : upper_neighbor_size_;
}
protected:
union UpperNeighborIndexMeta {
@@ -808,7 +812,8 @@ class HnswMmapStreamerEntity : public HnswStreamerEntity {
//! static_cast<const HnswMmapStreamerEntity&> in the algorithm is safe.
const HnswEntity::Pointer clone() const override;
inline TypedNeighbors get_neighbors_typed(level_t level, node_id_t id) const {
ailego_force_inline TypedNeighbors get_neighbors_typed(level_t level,
node_id_t id) const {
if (level == 0UL) {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset =
@@ -831,8 +836,9 @@ class HnswMmapStreamerEntity : public HnswStreamerEntity {
return TypedNeighbors(std::move(block));
}
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
ailego_force_inline int get_vector_typed(
const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
uint32_t chunk_idx = ids[i] >> node_index_mask_bits_;
@@ -843,7 +849,7 @@ class HnswMmapStreamerEntity : public HnswStreamerEntity {
return 0;
}
inline key_t get_key_typed(node_id_t id) const {
ailego_force_inline key_t get_key_typed(node_id_t id) const {
if (!use_key_info_map_) {
return id;
}
@@ -853,9 +859,18 @@ class HnswMmapStreamerEntity : public HnswStreamerEntity {
return *reinterpret_cast<const key_t *>(base + offset);
}
private:
//! Direct vector pointer access (no MemoryBlock wrapper).
//! For use in the merged search loop to avoid intermediate allocations.
ailego_force_inline const void *get_vector_ptr(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size();
return get_node_chunk_base(chunk_idx) + offset;
}
protected:
//! Get cached base address for a node chunk, syncing if needed
inline const char *get_node_chunk_base(uint32_t chunk_idx) const {
ailego_force_inline const char *get_node_chunk_base(
uint32_t chunk_idx) const {
if (ailego_unlikely(chunk_idx >= node_chunk_bases_.size())) {
sync_node_chunk_bases(chunk_idx);
}
@@ -863,7 +878,8 @@ class HnswMmapStreamerEntity : public HnswStreamerEntity {
}
//! Get cached base address for an upper neighbor chunk, syncing if needed
inline const char *get_upper_neighbor_chunk_base(uint32_t chunk_idx) const {
ailego_force_inline const char *get_upper_neighbor_chunk_base(
uint32_t chunk_idx) const {
if (ailego_unlikely(chunk_idx >= upper_neighbor_chunk_bases_.size())) {
sync_upper_neighbor_chunk_bases(chunk_idx);
}
@@ -926,9 +942,11 @@ class HnswBufferPoolStreamerEntity : public HnswStreamerEntity {
};
//! Typed entity subclass for contiguous memory mode.
//! Allocates contiguous memory (with hugepage/THP support) and copies all
//! chunk data into it. Access is via a single base pointer + offset,
//! eliminating chunk-level indirection and maximizing memory locality.
//! Splits node data into two dense arrays during build:
//! 1. vector_base_: flat vector array (stride = vector_size)
//! 2. graph_base_: key + L0 neighbors (stride = graph_stride_)
//! Total memory = vector_size + graph_stride_ per node (same as original
//! node_size), but each access pattern gets optimal cache locality.
class HnswContiguousStreamerEntity : public HnswMmapStreamerEntity {
public:
using HnswMmapStreamerEntity::HnswMmapStreamerEntity;
@@ -950,8 +968,10 @@ class HnswContiguousStreamerEntity : public HnswMmapStreamerEntity {
//! Degrade to mmap mode by releasing contiguous memory and falling back
//! to chunk-based access.
void degrade_to_mmap() {
node_memory_.reset();
node_base_ = nullptr;
vector_memory_.reset();
vector_base_ = nullptr;
graph_memory_.reset();
graph_base_ = nullptr;
upper_neighbor_memory_.reset();
upper_neighbor_base_ = nullptr;
upper_chunk_offsets_.clear();
@@ -959,7 +979,7 @@ class HnswContiguousStreamerEntity : public HnswMmapStreamerEntity {
}
bool is_contiguous() const {
return node_base_ != nullptr;
return vector_base_ != nullptr;
}
int add_vector(level_t level, key_t key, const void *vec,
@@ -974,11 +994,14 @@ class HnswContiguousStreamerEntity : public HnswMmapStreamerEntity {
return HnswMmapStreamerEntity::add_vector_with_id(level, id, vec);
}
inline TypedNeighbors get_neighbors_typed(level_t level, node_id_t id) const {
if (ailego_likely(node_base_ != nullptr)) {
ailego_force_inline TypedNeighbors get_neighbors_typed(level_t level,
node_id_t id) const {
if (ailego_likely(graph_base_ != nullptr)) {
if (level == 0UL) {
const char *ptr = node_base_ + static_cast<size_t>(id) * node_size() +
vector_size() + sizeof(key_t);
// graph layout: [key (sizeof(key_t)) | NeighborsHeader + neighbors]
const char *ptr = graph_base_ +
static_cast<size_t>(id) * graph_stride_ +
sizeof(key_t);
MmapMemoryBlock block(const_cast<char *>(ptr));
return TypedNeighbors(std::move(block));
}
@@ -999,13 +1022,14 @@ class HnswContiguousStreamerEntity : public HnswMmapStreamerEntity {
return HnswMmapStreamerEntity::get_neighbors_typed(level, id);
}
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
if (ailego_likely(node_base_ != nullptr)) {
ailego_force_inline int get_vector_typed(
const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
if (ailego_likely(vector_base_ != nullptr)) {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
const char *ptr =
node_base_ + static_cast<size_t>(ids[i]) * node_size();
vector_base_ + static_cast<size_t>(ids[i]) * vector_size();
vec_blocks[i].reset(const_cast<char *>(ptr));
}
return 0;
@@ -1013,18 +1037,29 @@ class HnswContiguousStreamerEntity : public HnswMmapStreamerEntity {
return HnswMmapStreamerEntity::get_vector_typed(ids, count, vec_blocks);
}
inline key_t get_key_typed(node_id_t id) const {
if (ailego_likely(node_base_ != nullptr)) {
ailego_force_inline key_t get_key_typed(node_id_t id) const {
if (ailego_likely(graph_base_ != nullptr)) {
if (!use_key_info_map_) {
return id;
}
const char *ptr =
node_base_ + static_cast<size_t>(id) * node_size() + vector_size();
const char *ptr = graph_base_ + static_cast<size_t>(id) * graph_stride_;
return *reinterpret_cast<const key_t *>(ptr);
}
return HnswMmapStreamerEntity::get_key_typed(id);
}
//! Direct vector pointer from flat vector array (stride = vector_size).
//! For use in the merged search loop to avoid intermediate allocations.
ailego_force_inline const void *get_vector_ptr(node_id_t id) const {
if (ailego_likely(vector_base_ != nullptr)) {
return vector_base_ + static_cast<size_t>(id) * vector_size();
}
// Fallback to mmap chunk-based access
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size();
return get_node_chunk_base(chunk_idx) + offset;
}
protected:
//! Custom deleter for contiguous memory (munmap / _aligned_free / free)
//! Used by shared_ptr to properly release mmap'd memory.
@@ -1042,12 +1077,17 @@ class HnswContiguousStreamerEntity : public HnswMmapStreamerEntity {
}
};
//! Shared ownership of contiguous memory (enables zero-copy clone)
std::shared_ptr<char> node_memory_{};
std::shared_ptr<char> upper_neighbor_memory_{};
//! Flat vector array: vectors stored densely (stride = vector_size).
std::shared_ptr<char> vector_memory_{};
char *vector_base_{nullptr};
//! Raw pointers for hot-path access (derived from shared_ptr)
char *node_base_{nullptr};
//! Graph array: [key | L0 neighbors] stored densely (stride = graph_stride_).
std::shared_ptr<char> graph_memory_{};
char *graph_base_{nullptr};
size_t graph_stride_{0}; // sizeof(key_t) + neighbor_size_
//! Shared ownership of upper neighbor contiguous memory
std::shared_ptr<char> upper_neighbor_memory_{};
char *upper_neighbor_base_{nullptr};
//! Cumulative offsets for each upper neighbor chunk in contiguous memory
+198 -50
View File
@@ -13,6 +13,8 @@
// limitations under the License.
#include "vamana_algorithm.h"
#include <type_traits>
#include <ailego/internal/cpu_features.h>
namespace zvec {
namespace core {
@@ -48,7 +50,8 @@ int VamanaAlgorithm<EntityType>::add_node(node_id_t id, VamanaContext *ctx) {
ctx->topk_heap().limit(search_list_size);
ctx->dist_calculator().clear_compare_cnt();
// Set query to the new node's vector
// Set query to the new node's vector. Use reset_query (same as search path)
// so that greedy_search works with the search-optimized distance kernel.
const void *query_vec = entity_.get_vector(id);
if (ailego_unlikely(query_vec == nullptr)) {
LOG_ERROR("Failed to get vector for node %u", id);
@@ -56,10 +59,11 @@ int VamanaAlgorithm<EntityType>::add_node(node_id_t id, VamanaContext *ctx) {
}
ctx->reset_query(query_vec);
greedy_search(entry_point, ctx);
greedy_search(entry_point, ctx, /*use_pool=*/false);
auto &topk_heap = ctx->topk_heap();
// Step 2: RobustPrune to select diverse neighbors
auto &topk_heap = ctx->topk_heap();
robust_prune(id, topk_heap, entity_.alpha(), entity_.max_degree(), ctx);
// Copy result before reverse updates (which also call robust_prune)
auto pruned_neighbors = ctx->prune_result();
@@ -96,38 +100,119 @@ int VamanaAlgorithm<EntityType>::search(VamanaContext *ctx) const {
uint32_t ef_search = std::max(static_cast<uint32_t>(ctx->topk()), ctx->ef());
topk_heap.limit(ef_search);
greedy_search(entry_point, ctx);
greedy_search(entry_point, ctx, /*use_pool=*/true);
return 0;
}
// ============================================================================
// greedy_search: Beam search from entry_point.
// greedy_search helper templates
//
// Maintains a candidate min-heap (ordered by distance) and a visited set.
// At each step, pops the closest unvisited candidate, expands its neighbors,
// and adds unvisited neighbors to both the candidate heap and the topk heap.
// Stops when the closest candidate is farther than the worst in topk, or
// when the scan limit is reached.
// Two specialized inner loops, dispatched from greedy_search():
//
// fast_greedy_search: mmap/contiguous with direct vector pointers.
// Uses batch_dist on a pointer array.
// slow_greedy_search: BufferPool-backed storage: must fetch MemBlock
// wrappers via get_vector_typed to pin pages.
//
// Both accept either BlockHeap or LinearPool as `HeapType` because the
// two expose the same reset(n, ef, block_size) / push_block(dists, ids, n)
// surface (LinearPool adapts via push_block and ignores the block_size hint).
// ============================================================================
template <typename EntityType>
void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
VamanaContext *ctx) const {
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
VamanaDistCalculator &dc = ctx->dist_calculator();
// mmap/contiguous variant: resolve vectors via get_vector_ptr
// and dispatch to the classic pointer-array batch_dist.
template <typename EntityType, typename HeapType>
void fast_greedy_search(const EntityType &entity, HeapType &pool,
VisitFilter &visit, VamanaDistCalculator &dc,
uint32_t topk, uint32_t ef, node_id_t entry_point,
uint32_t prefetch_lines) {
const uint32_t max_deg = entity.max_degree();
const uint32_t cap = std::max(topk, ef);
pool.reset(static_cast<int32_t>(cap), static_cast<int32_t>(max_deg));
visit.clear();
dist_t ep_dist = dc.batch_dist(entry_point);
visit.set_visited(entry_point);
pool.push_block(&ep_dist, &entry_point, 1);
static constexpr uint32_t GRAPH_PO = 8;
uint32_t buf_capacity = max_deg;
std::vector<node_id_t> neighbor_ids(buf_capacity);
std::vector<float> dists(buf_capacity);
std::vector<const void *> neighbor_vecs(buf_capacity);
while (pool.has_next()) {
auto current_node = pool.pop();
const auto neighbors = entity.get_neighbors_typed(current_node);
ailego_prefetch(neighbors.data);
if (neighbors.size() > buf_capacity) {
buf_capacity = neighbors.size();
neighbor_ids.resize(buf_capacity);
dists.resize(buf_capacity);
neighbor_vecs.resize(buf_capacity);
}
const uint32_t po =
std::min(static_cast<uint32_t>(neighbors.size()), GRAPH_PO);
uint32_t unvisited_count = 0;
uint32_t i = 0;
for (; i < po; ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) continue;
visit.set_visited(node);
const void *vec_ptr = entity.get_vector_ptr(node);
const char *p = reinterpret_cast<const char *>(vec_ptr);
for (uint32_t cl = 0; cl < prefetch_lines; ++cl) {
ailego_prefetch(p + cl * 64);
}
neighbor_ids[unvisited_count] = node;
neighbor_vecs[unvisited_count] = vec_ptr;
unvisited_count++;
}
for (; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
if (visit.visited(node)) continue;
visit.set_visited(node);
neighbor_ids[unvisited_count] = node;
neighbor_vecs[unvisited_count] = entity.get_vector_ptr(node);
unvisited_count++;
}
if (unvisited_count == 0) continue;
dc.batch_dist(neighbor_vecs.data(), unvisited_count, dists.data());
pool.push_block(dists.data(), neighbor_ids.data(),
static_cast<int32_t>(unvisited_count));
}
}
// ============================================================================
// dual_heap_greedy_search: shared core for the fallback dual-heap path.
//
// Maintains a candidate min-heap + topk heap + VisitFilter. Uses plain
// batch_dist.
// ============================================================================
template <typename EntityType, typename MemBlockType, typename FilterFn>
void dual_heap_greedy_search(const EntityType &entity, VamanaContext *ctx,
VamanaDistCalculator &dc, node_id_t entry_point,
FilterFn &&filter) {
static constexpr uint32_t PREFETCH_BATCH = 2;
static constexpr uint32_t PREFETCH_STEP = 2;
uint32_t buf_capacity = entity.max_degree();
std::vector<node_id_t> neighbor_ids(buf_capacity);
std::vector<MemBlockType> neighbor_vec_blocks;
neighbor_vec_blocks.reserve(buf_capacity);
std::vector<float> dists(buf_capacity);
std::vector<const void *> neighbor_vecs(buf_capacity);
VisitFilter &visit = ctx->visit_filter();
CandidateHeap &candidates = ctx->candidates();
auto &topk_heap = ctx->topk_heap();
const IndexFilter &index_filter =
static_cast<const IndexContext *>(ctx)->filter();
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
if (index_filter.is_valid()) {
filter = [&](node_id_t id) {
return index_filter(entity.get_key_typed(id));
};
}
candidates.clear();
visit.clear();
@@ -147,16 +232,6 @@ void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
}
candidates.emplace(entry_point, entry_dist);
// Pre-allocate temporary vectors outside the hot loop to avoid
// per-iteration heap allocations. Sized to max_degree initially;
// resized inside the loop if actual neighbor count exceeds this.
uint32_t buf_capacity = entity.max_degree();
std::vector<node_id_t> neighbor_ids(buf_capacity);
std::vector<MemBlockType> neighbor_vec_blocks;
neighbor_vec_blocks.reserve(buf_capacity);
std::vector<float> dists(buf_capacity);
std::vector<const void *> neighbor_vecs(buf_capacity);
while (!candidates.empty() && !ctx->reach_scan_limit()) {
auto top = candidates.begin();
node_id_t current_node = top->first;
@@ -198,28 +273,17 @@ void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
neighbor_vec_blocks);
if (ailego_unlikely(ret != 0)) break;
// Prefetch for better cache performance
static constexpr uint32_t PREFETCH_BATCH = 2;
static constexpr uint32_t PREFETCH_STEP = 2;
for (uint32_t i = 0;
i < std::min(PREFETCH_BATCH * PREFETCH_STEP, unvisited_count); ++i) {
ailego_prefetch(neighbor_vec_blocks[i].data());
}
// Batch distance computation (reuse pre-allocated buffers)
// Batch distance computation (reuse pre-allocated buffers).
for (uint32_t i = 0; i < unvisited_count; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
}
dc.batch_dist(neighbor_vecs.data(), unvisited_count, dists.data());
// Update candidates and topk.
// Unlike vanilla DiskANN which inserts all unvisited neighbors into
// the candidate queue unconditionally, we apply an early-pruning
// optimization: a neighbor is only inserted into the candidate queue
// (and topk_heap) if it could potentially improve the final results,
// i.e. either the topk heap is not yet full, or the neighbor is closer
// than the current worst result. This avoids expanding clearly
// unpromising branches and reduces the candidate queue size.
for (uint32_t i = 0; i < unvisited_count; ++i) {
node_id_t node = neighbor_ids[i];
dist_t node_dist = dists[i];
@@ -233,6 +297,88 @@ void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
}
}
// ============================================================================
// greedy_search: Beam search from entry_point.
//
// Maintains a candidate min-heap (ordered by distance) and a visited set.
// At each step, pops the closest unvisited candidate, expands its neighbors,
// and adds unvisited neighbors to both the candidate heap and the topk heap.
// Stops when the closest candidate is farther than the worst in topk, or
// when the scan limit is reached.
// ============================================================================
template <typename EntityType>
void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
VamanaContext *ctx,
bool use_pool) const {
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
VamanaDistCalculator &dc = ctx->dist_calculator();
const IndexFilter &index_filter =
static_cast<const IndexContext *>(ctx)->filter();
// Number of cache lines per vector (e.g. 2 for dim=128).
// Used by both the fallback candidates/filter path and the fast helpers.
uint32_t prefetch_lines = (dc.dimension() + 63) / 64;
if constexpr (std::is_same_v<EntityType, VamanaContiguousStreamerEntity>) {
// Contiguous flat array stride is already 64B-aligned. Use it so that
// prefetch does not overshoot into the next vector.
size_t stride = entity.vector_stride();
if (stride > 0) {
prefetch_lines = static_cast<uint32_t>(stride / 64);
}
}
if (!use_pool || index_filter.is_valid()) {
// Fallback path used by add_node (use_pool=false) and filtered search.
// Dispatched to dual_heap_greedy_search (plain batch_dist).
auto run_with_filter = [&](auto &&filter) {
dual_heap_greedy_search<EntityType, MemBlockType>(
entity, ctx, dc, entry_point, std::forward<decltype(filter)>(filter));
};
if (index_filter.is_valid()) {
auto filter = [&](node_id_t id) {
return index_filter(entity.get_key_typed(id));
};
run_with_filter(filter);
} else {
auto filter = [](node_id_t) { return false; };
run_with_filter(filter);
}
} else {
// Fast pool-based path for mmap/contiguous entities that support
// direct pointer access. BlockHeap (AVX2) or LinearPool (scalar)
// are used for top-k tracking. BufferPool entities fall back to
// dual_heap_greedy_search since they lack direct pointer access.
if constexpr (std::is_same_v<MemBlockType, MmapMemoryBlock>) {
const uint32_t topk_v = static_cast<uint32_t>(ctx->topk());
const uint32_t ef_v = ctx->ef();
const bool avx2_ok =
zvec::ailego::internal::CpuFeatures::static_flags_.AVX2;
auto &topk_heap = ctx->topk_heap();
auto &visit = ctx->visit_filter();
if (avx2_ok) {
auto &bpool = ctx->block_pool();
fast_greedy_search(entity, bpool, visit, dc, topk_v, ef_v, entry_point,
prefetch_lines);
copy_pool_to_topk(bpool, topk_heap);
} else {
auto &lpool = ctx->pool();
fast_greedy_search(entity, lpool, visit, dc, topk_v, ef_v, entry_point,
prefetch_lines);
copy_pool_to_topk(lpool, topk_heap);
}
} else {
// BufferPool entities: fallback to dual-heap path.
auto filter = [](node_id_t) { return false; };
dual_heap_greedy_search<EntityType, MemBlockType>(entity, ctx, dc,
entry_point, filter);
}
}
}
// ============================================================================
// robust_prune: Select up to max_degree diverse neighbors from candidates.
//
@@ -334,9 +480,11 @@ void VamanaAlgorithm<EntityType>::robust_prune(node_id_t id,
}
if (batch_count > 0) {
// Batch compute distances from selected candidate to remaining
dc.batch_dist_pair(selected_vec, batch_vecs.data(), batch_count,
batch_dists.data());
// Compute distances from selected candidate to remaining candidates.
// distance_ is the symmetric data-to-data kernel (no pairwise split).
for (uint32_t k = 0; k < batch_count; ++k) {
batch_dists[k] = dc.dist(selected_vec, batch_vecs[k]);
}
// DiskANN (L2/Cosine):
// occlude_factor[t] = max(occlude_factor[t], dist_to_query /
+2 -1
View File
@@ -80,7 +80,8 @@ class VamanaAlgorithm : public VamanaAlgorithmBase {
// GreedySearch: starting from entry_point, greedily expand the closest
// unvisited candidate until the search list is exhausted or scan limit
// is reached. Results accumulate in topk_heap.
void greedy_search(node_id_t entry_point, VamanaContext *ctx) const;
void greedy_search(node_id_t entry_point, VamanaContext *ctx,
bool use_pool) const;
// RobustPrune: given a candidate set (topk_heap), select up to max_degree
// diverse neighbors using alpha-based distance comparison.
+13 -6
View File
@@ -14,6 +14,8 @@
#pragma once
#include <zvec/core/framework/index_context.h>
#include "utility/block_heap.h"
#include "utility/linear_pool.h"
#include "utility/visit_filter.h"
#include "vamana_dist_calculator.h"
#include "vamana_entity.h"
@@ -119,6 +121,14 @@ class VamanaContext : public IndexContext {
inline TopkHeap &update_heap() {
return update_heap_;
}
inline LinearPool<dist_t> &pool() {
return pool_;
}
// Block-insert pool used by the AVX2-gated greedy_search fast path.
// Only accessed under a runtime CpuFeatures::AVX2 guard at call sites.
inline BlockHeap &block_pool() {
return block_pool_;
}
inline VisitFilter &visit_filter() {
return visit_filter_;
}
@@ -256,12 +266,6 @@ class VamanaContext : public IndexContext {
return topk_;
}
inline void update_dist_caculator_distance(
const IndexMetric::MatrixDistance &distance,
const IndexMetric::MatrixBatchDistance &batch_distance) {
dc_.update_distance(distance, batch_distance);
}
private:
void fill_random_to_topk_full(void);
@@ -320,6 +324,9 @@ class VamanaContext : public IndexContext {
VisitFilter::Mode filter_mode_{VisitFilter::ByteMap};
float filter_negative_prob_{VamanaEntity::kDefaultBFNegativeProbability};
LinearPool<dist_t> pool_;
BlockHeap block_pool_;
};
} // namespace core
@@ -65,13 +65,6 @@ class VamanaDistCalculator {
dim_ = dim;
}
inline void update_distance(
const IndexMetric::MatrixDistance &distance,
const IndexMetric::MatrixBatchDistance &batch_distance) {
distance_ = distance;
batch_distance_ = batch_distance;
}
inline void reset_query(const void *query) {
error_ = false;
query_ = query;
@@ -136,22 +129,6 @@ class VamanaDistCalculator {
return score;
}
// Batch distance computation between a base vector and multiple target
// vectors. Does NOT use query_ and does NOT increment compare_cnt. Used for
// inter-candidate distance computation in robust_prune.
//
// Uses the single distance function (distance_) in a loop rather than
// batch_distance_, because batch_distance_ (turbo AVX512-VNNI) expects
// the second argument to be a preprocessed uint8 query (+128 shift),
// while base_vec here is a raw int8 stored vector. The single distance
// function (AVX2 sign/abs trick) correctly handles two raw int8 inputs.
inline void batch_dist_pair(const void *base_vec, const void **vecs,
uint32_t count, float *dists) {
for (uint32_t i = 0; i < count; ++i) {
distance_(base_vec, vecs[i], dim_, &dists[i]);
}
}
dist_t operator()(const void *vec) {
return dist(vec);
}
+6 -18
View File
@@ -235,6 +235,12 @@ int VamanaStreamer::open(IndexStorage::Pointer stg) {
auto metric_params = index_meta.metric_params();
metric_params.merge(meta_.metric_params());
meta_.set_metric(index_meta.metric_name(), 0, metric_params);
// Propagate reformer info from stored meta (needed for quantizers
// whose reformer params are computed during training, e.g. UniformInt8)
if (!index_meta.reformer_name().empty()) {
meta_.set_reformer(index_meta.reformer_name(), 0,
index_meta.reformer_params());
}
}
// Create metric
@@ -256,17 +262,6 @@ int VamanaStreamer::open(IndexStorage::Pointer stg) {
return IndexError_InvalidArgument;
}
add_distance_ = metric_->distance();
add_batch_distance_ = metric_->batch_distance();
search_distance_ = add_distance_;
search_batch_distance_ = add_batch_distance_;
if (metric_->query_metric() && metric_->query_metric()->distance() &&
metric_->query_metric()->batch_distance()) {
search_distance_ = metric_->query_metric()->distance();
search_batch_distance_ = metric_->query_metric()->batch_distance();
}
// Create algorithm based on entity storage mode
switch (entity_->storage_mode()) {
case VamanaStorageMode::kBufferPool:
@@ -456,8 +451,6 @@ int VamanaStreamer::add_impl(uint64_t pkey, const void *query,
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
if (metric_->support_train()) {
@@ -529,8 +522,6 @@ int VamanaStreamer::add_with_id_impl(uint32_t id, const void *query,
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
ctx->clear();
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
ctx->reset_query(query);
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
if (metric_->support_train()) {
@@ -593,7 +584,6 @@ int VamanaStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
@@ -655,7 +645,6 @@ int VamanaStreamer::search_bf_impl(const void *query,
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
const auto &filter = static_cast<IndexContext *>(ctx)->filter();
@@ -697,7 +686,6 @@ int VamanaStreamer::search_bf_by_p_keys_impl(
}
ctx->clear();
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
ctx->resize_results(count);
auto &topk = ctx->topk_heap();
@@ -150,11 +150,6 @@ class VamanaStreamer : public IndexStreamer {
IndexMeta meta_{};
IndexMetric::Pointer metric_{};
IndexMetric::MatrixDistance add_distance_{};
IndexMetric::MatrixDistance search_distance_{};
IndexMetric::MatrixBatchDistance add_batch_distance_{};
IndexMetric::MatrixBatchDistance search_batch_distance_{};
Stats stats_{};
std::mutex mutex_{};
@@ -12,9 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "vamana_streamer_entity.h"
#if defined(__linux__) || defined(__APPLE__)
#include <sys/mman.h>
#endif
#include <ailego/utility/memory_helper.h>
#include <zvec/ailego/hash/crc32c.h>
#include <zvec/core/framework/index_stats.h>
@@ -540,8 +537,12 @@ const VamanaEntity::Pointer VamanaContiguousStreamerEntity::clone() const {
}
// Share contiguous memory with the clone (zero-copy)
entity->node_memory_ = node_memory_;
entity->node_base_ = node_base_;
entity->vector_memory_ = vector_memory_;
entity->vector_base_ = vector_base_;
entity->vector_stride_ = vector_stride_;
entity->graph_memory_ = graph_memory_;
entity->graph_base_ = graph_base_;
entity->graph_stride_ = graph_stride_;
return VamanaEntity::Pointer(entity);
}
@@ -552,56 +553,60 @@ const VamanaEntity::Pointer VamanaContiguousStreamerEntity::clone() const {
char *VamanaContiguousStreamerEntity::allocate_contiguous(size_t size) {
if (size == 0) return nullptr;
#if defined(__linux__)
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (ptr == MAP_FAILED) {
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
return nullptr;
}
::madvise(ptr, size, MADV_HUGEPAGE);
return static_cast<char *>(ptr);
#elif defined(__APPLE__)
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANON, -1, 0);
if (ptr == MAP_FAILED) {
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
return nullptr;
}
return static_cast<char *>(ptr);
#elif defined(_WIN32)
void *ptr = ::_aligned_malloc(size, ailego::MemoryHelper::PageSize());
void *ptr = ailego::MemoryHelper::AllocateHugePage(size);
if (!ptr) {
LOG_ERROR("_aligned_malloc failed for contiguous memory, size=%zu", size);
LOG_ERROR("AllocateHugePage failed for contiguous memory, size=%zu", size);
return nullptr;
}
return static_cast<char *>(ptr);
#else
void *ptr = std::aligned_alloc(ailego::MemoryHelper::PageSize(), size);
if (!ptr) {
LOG_ERROR("aligned_alloc failed, size=%zu", size);
return nullptr;
}
return static_cast<char *>(ptr);
#endif
}
int VamanaContiguousStreamerEntity::build_contiguous_memory() {
node_memory_.reset();
node_base_ = nullptr;
vector_memory_.reset();
vector_base_ = nullptr;
vector_stride_ = 0;
graph_memory_.reset();
graph_base_ = nullptr;
const uint32_t total_docs = doc_cnt();
if (total_docs == 0) return 0;
const size_t per_node = node_size();
const size_t total_node_data = static_cast<size_t>(total_docs) * per_node;
size_t node_memory_size = AlignHugePageSize(total_node_data);
char *raw_node = allocate_contiguous(node_memory_size);
if (!raw_node) return IndexError_Runtime;
node_memory_.reset(raw_node, ContiguousDeleter{node_memory_size});
node_base_ = raw_node;
const size_t vec_size = vector_size();
// Copy node data from chunks into contiguous memory
// Pad per-vector stride up to kVectorAlignment (64B) so every vector
// starts on a cache-line boundary.
vector_stride_ =
(vec_size + (kVectorAlignment - 1)) & ~(kVectorAlignment - 1);
// graph_stride = key + neighbors (everything except vector)
graph_stride_ = sizeof(key_t) + neighbors_size();
// Allocate flat vector array (stride = vector_stride_, padded for 64B)
const size_t total_vec_data =
static_cast<size_t>(total_docs) * vector_stride_;
size_t vector_memory_size = AlignHugePageSize(total_vec_data);
char *raw_vec = allocate_contiguous(vector_memory_size);
if (!raw_vec) return IndexError_Runtime;
vector_memory_.reset(raw_vec, ContiguousDeleter{vector_memory_size});
vector_base_ = raw_vec;
// Allocate graph array (stride = sizeof(key_t) + neighbors_size)
const size_t total_graph_data =
static_cast<size_t>(total_docs) * graph_stride_;
size_t graph_memory_size = AlignHugePageSize(total_graph_data);
char *raw_graph = allocate_contiguous(graph_memory_size);
if (!raw_graph) {
vector_memory_.reset();
vector_base_ = nullptr;
vector_stride_ = 0;
return IndexError_Runtime;
}
graph_memory_.reset(raw_graph, ContiguousDeleter{graph_memory_size});
graph_base_ = raw_graph;
// Split node data from chunks into vector / graph arrays.
// Original node layout: [vector (vec_size) | key (8B) | neighbors]
// Padding bytes in vector_base_ are left zero (anon mmap is zero-filled).
const auto &chunks = node_chunks_;
const uint32_t nodes_per_chunk = 1U << node_index_mask_bits_;
for (size_t chunk_idx = 0; chunk_idx < chunks.size(); ++chunk_idx) {
@@ -613,14 +618,27 @@ int VamanaContiguousStreamerEntity::build_contiguous_memory() {
uint32_t count_in_chunk = std::min(nodes_per_chunk, total_docs - base_id);
const char *src = static_cast<const char *>(chunk_data);
char *dst = node_base_ + static_cast<size_t>(base_id) * per_node;
std::memcpy(dst, src, static_cast<size_t>(count_in_chunk) * per_node);
for (uint32_t i = 0; i < count_in_chunk; ++i) {
const char *node_src = src + static_cast<size_t>(i) * per_node;
size_t global_id = static_cast<size_t>(base_id + i);
// Copy vector to flat vector array at padded stride
std::memcpy(vector_base_ + global_id * vector_stride_, node_src,
vec_size);
// Copy key + neighbors to graph array
std::memcpy(graph_base_ + global_id * graph_stride_, node_src + vec_size,
graph_stride_);
}
}
LOG_INFO(
"Built Vamana contiguous memory: node_size=%zu total_docs=%u "
"node_chunks=%zu",
node_memory_size, total_docs, chunks.size());
"Built Vamana contiguous memory: "
"vector_mem=%zu graph_mem=%zu total_docs=%u "
"node_chunks=%zu vector_size=%zu vector_stride=%zu "
"(cache-line aligned to %zuB)",
vector_memory_size, graph_memory_size, total_docs, chunks.size(),
vec_size, vector_stride_, kVectorAlignment);
return 0;
}
@@ -16,10 +16,8 @@
#include <iostream>
#include <memory>
#include <mutex>
#if defined(__linux__) || defined(__APPLE__)
#include <sys/mman.h>
#endif
#include <ailego/parallel/lock.h>
#include <ailego/utility/memory_helper.h>
#include <sparsehash/dense_hash_map>
#include <zvec/ailego/container/heap.h>
#include <zvec/core/framework/index_framework.h>
@@ -478,7 +476,7 @@ class VamanaMmapStreamerEntity : public VamanaStreamerEntity {
//! static_cast<const VamanaMmapStreamerEntity&> in the algorithm is safe.
const VamanaEntity::Pointer clone() const override;
inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
ailego_force_inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset =
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
@@ -487,8 +485,9 @@ class VamanaMmapStreamerEntity : public VamanaStreamerEntity {
return TypedNeighbors(std::move(block));
}
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
ailego_force_inline int get_vector_typed(
const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
uint32_t chunk_idx = ids[i] >> node_index_mask_bits_;
@@ -499,7 +498,7 @@ class VamanaMmapStreamerEntity : public VamanaStreamerEntity {
return 0;
}
inline key_t get_key_typed(node_id_t id) const {
ailego_force_inline key_t get_key_typed(node_id_t id) const {
if (!use_key_info_map_) return id;
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
@@ -507,8 +506,17 @@ class VamanaMmapStreamerEntity : public VamanaStreamerEntity {
return *reinterpret_cast<const key_t *>(base + offset);
}
//! Direct vector pointer access (no MemoryBlock wrapper).
//! For use in the merged search loop to avoid intermediate allocations.
ailego_force_inline const void *get_vector_ptr(node_id_t id) const {
uint32_t chunk_idx = id >> node_index_mask_bits_;
uint32_t offset = (id & node_index_mask_) * node_size();
return get_node_chunk_base(chunk_idx) + offset;
}
private:
inline const char *get_node_chunk_base(uint32_t chunk_idx) const {
ailego_force_inline const char *get_node_chunk_base(
uint32_t chunk_idx) const {
if (ailego_unlikely(chunk_idx >= node_chunk_bases_.size())) {
sync_node_chunk_bases(chunk_idx);
}
@@ -569,9 +577,11 @@ class VamanaBufferPoolStreamerEntity : public VamanaStreamerEntity {
};
// --- Typed entity subclass for contiguous memory mode ---
// Allocates contiguous memory and copies all chunk data into it.
// Access is via a single base pointer + offset, eliminating chunk-level
// indirection and maximizing memory locality.
// Splits node data into two dense arrays during build:
// 1. vector_base_: flat vector array (stride = vector_size)
// 2. graph_base_: key + neighbors (stride = graph_stride_)
// Total memory = vector_size + graph_stride_ per node (same as original
// node_size), but each access pattern gets optimal cache locality.
class VamanaContiguousStreamerEntity : public VamanaMmapStreamerEntity {
public:
using VamanaMmapStreamerEntity::VamanaMmapStreamerEntity;
@@ -592,13 +602,23 @@ class VamanaContiguousStreamerEntity : public VamanaMmapStreamerEntity {
//! Degrade to mmap mode by releasing contiguous memory and falling back
//! to chunk-based access.
void degrade_to_mmap() {
node_memory_.reset();
node_base_ = nullptr;
vector_memory_.reset();
vector_base_ = nullptr;
vector_stride_ = 0;
graph_memory_.reset();
graph_base_ = nullptr;
LOG_INFO("Vamana contiguous entity degraded to mmap mode for insertion");
}
bool is_contiguous() const {
return node_base_ != nullptr;
return vector_base_ != nullptr;
}
//! Per-entry stride of the flat vector array (0 if no contiguous build).
//! Padded up to kVectorAlignment (64B), so it is also the amount that
//! should be prefetched per vector.
size_t vector_stride() const {
return vector_stride_;
}
int add_vector(key_t key, const void *vec, node_id_t *id) override {
@@ -611,23 +631,25 @@ class VamanaContiguousStreamerEntity : public VamanaMmapStreamerEntity {
return VamanaMmapStreamerEntity::add_vector_with_id(id, vec);
}
inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
if (ailego_likely(node_base_ != nullptr)) {
const char *ptr = node_base_ + static_cast<size_t>(id) * node_size() +
vector_size() + sizeof(key_t);
ailego_force_inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
if (ailego_likely(graph_base_ != nullptr)) {
// graph layout: [key (sizeof(key_t)) | NeighborsHeader + neighbors]
const char *ptr =
graph_base_ + static_cast<size_t>(id) * graph_stride_ + sizeof(key_t);
MmapMemoryBlock block(const_cast<char *>(ptr));
return TypedNeighbors(std::move(block));
}
return VamanaMmapStreamerEntity::get_neighbors_typed(id);
}
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
if (ailego_likely(node_base_ != nullptr)) {
ailego_force_inline int get_vector_typed(
const node_id_t *ids, uint32_t count,
std::vector<MmapMemoryBlock> &vec_blocks) const {
if (ailego_likely(vector_base_ != nullptr)) {
vec_blocks.resize(count);
for (auto i = 0U; i < count; ++i) {
const char *ptr =
node_base_ + static_cast<size_t>(ids[i]) * node_size();
vector_base_ + static_cast<size_t>(ids[i]) * vector_stride_;
vec_blocks[i].reset(const_cast<char *>(ptr));
}
return 0;
@@ -635,37 +657,53 @@ class VamanaContiguousStreamerEntity : public VamanaMmapStreamerEntity {
return VamanaMmapStreamerEntity::get_vector_typed(ids, count, vec_blocks);
}
inline key_t get_key_typed(node_id_t id) const {
if (ailego_likely(node_base_ != nullptr)) {
ailego_force_inline key_t get_key_typed(node_id_t id) const {
if (ailego_likely(graph_base_ != nullptr)) {
if (!use_key_info_map_) return id;
const char *ptr =
node_base_ + static_cast<size_t>(id) * node_size() + vector_size();
// key is at offset 0 within each graph node
const char *ptr = graph_base_ + static_cast<size_t>(id) * graph_stride_;
return *reinterpret_cast<const key_t *>(ptr);
}
return VamanaMmapStreamerEntity::get_key_typed(id);
}
//! Direct vector pointer from flat vector array.
//! Stride is padded up to kVectorAlignment (64B) to preserve cache-line
//! alignment even when vector_size is not a multiple of 64. The padding is
//! purely in-memory and does NOT affect the on-disk index file layout.
ailego_force_inline const void *get_vector_ptr(node_id_t id) const {
if (ailego_likely(vector_base_ != nullptr)) {
return vector_base_ + static_cast<size_t>(id) * vector_stride_;
}
return VamanaMmapStreamerEntity::get_vector_ptr(id);
}
protected:
//! Custom deleter for contiguous memory (munmap / _aligned_free / free)
//! Custom deleter for contiguous memory allocated via
//! MemoryHelper::AllocateHugePage. `size` is the (already huge-page-aligned)
//! length passed at allocation time, required by the mmap/munmap path.
struct ContiguousDeleter {
size_t size;
void operator()(char *ptr) const {
if (!ptr) return;
#if defined(__linux__) || defined(__APPLE__)
::munmap(ptr, size);
#elif defined(_WIN32)
::_aligned_free(ptr);
#else
std::free(ptr);
#endif
ailego::MemoryHelper::FreeHugePage(ptr, size);
}
};
//! Shared ownership of contiguous memory (enables zero-copy clone)
std::shared_ptr<char> node_memory_{};
//! Flat vector array: vectors stored densely with per-vector stride
//! padded up to kVectorAlignment (64B) to keep each vector's starting
//! address cache-line aligned. Base is page-aligned by the allocator.
std::shared_ptr<char> vector_memory_{};
char *vector_base_{nullptr};
//! Per-vector stride = AlignUp(vector_size(), kVectorAlignment).
size_t vector_stride_{0};
//! Raw pointer for hot-path access (derived from shared_ptr)
char *node_base_{nullptr};
//! Graph array: [key | neighbors] stored densely (stride = graph_stride_).
std::shared_ptr<char> graph_memory_{};
char *graph_base_{nullptr};
size_t graph_stride_{0}; // sizeof(key_t) + neighbors_size()
//! Cache-line alignment used for per-vector stride in the flat array.
static constexpr size_t kVectorAlignment = 64;
private:
static char *allocate_contiguous(size_t size);