feat: refac entity and impl Vamana. (#371)
This commit is contained in:
@@ -7,6 +7,7 @@ cc_directory(flat_sparse)
|
||||
cc_directory(ivf)
|
||||
cc_directory(hnsw)
|
||||
cc_directory(hnsw_sparse)
|
||||
cc_directory(vamana)
|
||||
if(RABITQ_SUPPORTED)
|
||||
message(STATUS "BUILD RABITQ")
|
||||
cc_directory(hnsw_rabitq)
|
||||
|
||||
@@ -12,28 +12,15 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "hnsw_algorithm.h"
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswAlgorithm::HnswAlgorithm(HnswEntity &entity)
|
||||
: entity_(entity),
|
||||
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
|
||||
lock_pool_(kLockCnt) {}
|
||||
|
||||
int HnswAlgorithm::cleanup() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswAlgorithm::add_node(node_id_t id, level_t level, HnswContext *ctx) {
|
||||
template <typename EntityType>
|
||||
int HnswAlgorithm<EntityType>::add_node(node_id_t id, level_t level,
|
||||
HnswContext *ctx) {
|
||||
spin_lock_.lock();
|
||||
|
||||
// std::cout << "id: " << id << ", level: " << level << std::endl;
|
||||
|
||||
auto cur_max_level = entity_.cur_max_level();
|
||||
auto entry_point = entity_.entry_point();
|
||||
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
|
||||
@@ -54,7 +41,7 @@ int HnswAlgorithm::add_node(node_id_t id, level_t level, HnswContext *ctx) {
|
||||
}
|
||||
|
||||
level_t cur_level = cur_max_level;
|
||||
dist_t dist = ctx->dist_calculator()(entry_point);
|
||||
dist_t dist = ctx->dist_calculator().batch_dist(entry_point);
|
||||
for (; cur_level > level; --cur_level) {
|
||||
select_entry_point(cur_level, &entry_point, &dist, ctx);
|
||||
}
|
||||
@@ -81,7 +68,8 @@ int HnswAlgorithm::add_node(node_id_t id, level_t level, HnswContext *ctx) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswAlgorithm::search(HnswContext *ctx) const {
|
||||
template <typename EntityType>
|
||||
int HnswAlgorithm<EntityType>::search(HnswContext *ctx) const {
|
||||
spin_lock_.lock();
|
||||
auto maxLevel = entity_.cur_max_level();
|
||||
auto entry_point = entity_.entry_point();
|
||||
@@ -107,13 +95,15 @@ int HnswAlgorithm::search(HnswContext *ctx) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! select_entry_point on hnsw level, ef = 1
|
||||
void HnswAlgorithm::select_entry_point(level_t level, node_id_t *entry_point,
|
||||
dist_t *dist, HnswContext *ctx) const {
|
||||
auto &entity = ctx->get_entity();
|
||||
template <typename EntityType>
|
||||
void HnswAlgorithm<EntityType>::select_entry_point(level_t level,
|
||||
node_id_t *entry_point,
|
||||
dist_t *dist,
|
||||
HnswContext *ctx) const {
|
||||
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
|
||||
HnswDistCalculator &dc = ctx->dist_calculator();
|
||||
while (true) {
|
||||
const Neighbors neighbors = entity.get_neighbors(level, *entry_point);
|
||||
const auto neighbors = entity.get_neighbors_typed(level, *entry_point);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
@@ -122,8 +112,8 @@ void HnswAlgorithm::select_entry_point(level_t level, node_id_t *entry_point,
|
||||
break;
|
||||
}
|
||||
|
||||
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
|
||||
int ret = entity.get_vector(&neighbors[0], size, neighbor_vec_blocks);
|
||||
std::vector<MemBlockType> neighbor_vec_blocks;
|
||||
int ret = entity.get_vector_typed(&neighbors[0], size, neighbor_vec_blocks);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_vector())++;
|
||||
}
|
||||
@@ -159,8 +149,10 @@ void HnswAlgorithm::select_entry_point(level_t level, node_id_t *entry_point,
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswAlgorithm::add_neighbors(node_id_t id, level_t level,
|
||||
TopkHeap &topk_heap, HnswContext *ctx) {
|
||||
template <typename EntityType>
|
||||
void HnswAlgorithm<EntityType>::add_neighbors(node_id_t id, level_t level,
|
||||
TopkHeap &topk_heap,
|
||||
HnswContext *ctx) {
|
||||
if (ailego_unlikely(topk_heap.size() == 0)) {
|
||||
return;
|
||||
}
|
||||
@@ -178,16 +170,20 @@ void HnswAlgorithm::add_neighbors(node_id_t id, level_t level,
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
|
||||
dist_t *dist, TopkHeap &topk,
|
||||
HnswContext *ctx) const {
|
||||
const auto &entity = ctx->get_entity();
|
||||
template <typename EntityType>
|
||||
void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
|
||||
node_id_t *entry_point,
|
||||
dist_t *dist, TopkHeap &topk,
|
||||
HnswContext *ctx) const {
|
||||
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
|
||||
HnswDistCalculator &dc = ctx->dist_calculator();
|
||||
VisitFilter &visit = ctx->visit_filter();
|
||||
CandidateHeap &candidates = ctx->candidates();
|
||||
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
|
||||
if (ctx->filter().is_valid()) {
|
||||
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
|
||||
filter = [&](node_id_t id) {
|
||||
return ctx->filter()(entity.get_key_typed(id));
|
||||
};
|
||||
}
|
||||
|
||||
candidates.clear();
|
||||
@@ -208,7 +204,7 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
|
||||
}
|
||||
|
||||
candidates.pop();
|
||||
const Neighbors neighbors = entity.get_neighbors(level, main_node);
|
||||
const auto neighbors = entity.get_neighbors_typed(level, main_node);
|
||||
ailego_prefetch(neighbors.data);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
@@ -231,8 +227,9 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
|
||||
int ret = entity.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
|
||||
std::vector<MemBlockType> neighbor_vec_blocks;
|
||||
int ret =
|
||||
entity.get_vector_typed(neighbor_ids.data(), size, neighbor_vec_blocks);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_vector())++;
|
||||
}
|
||||
@@ -278,15 +275,16 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
|
||||
HnswContext *ctx) const {
|
||||
template <typename EntityType>
|
||||
void HnswAlgorithm<EntityType>::expand_neighbors_by_group(
|
||||
TopkHeap &topk, HnswContext *ctx) const {
|
||||
if (!ctx->group_by().is_valid()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto &entity = ctx->get_entity();
|
||||
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
|
||||
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
|
||||
return ctx->group_by()(entity.get_key(id));
|
||||
return ctx->group_by()(entity.get_key_typed(id));
|
||||
};
|
||||
|
||||
// devide into groups
|
||||
@@ -312,7 +310,9 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
|
||||
|
||||
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
|
||||
if (ctx->filter().is_valid()) {
|
||||
filter = [&](node_id_t id) { return ctx->filter()(entity.get_key(id)); };
|
||||
filter = [&](node_id_t id) {
|
||||
return ctx->filter()(entity.get_key_typed(id));
|
||||
};
|
||||
}
|
||||
|
||||
// refill to get enough groups
|
||||
@@ -332,7 +332,7 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
|
||||
node_id_t main_node = top->first;
|
||||
|
||||
candidates.pop();
|
||||
const Neighbors neighbors = entity.get_neighbors(0, main_node);
|
||||
const auto neighbors = entity.get_neighbors_typed(0, main_node);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
@@ -354,9 +354,9 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
|
||||
int ret =
|
||||
entity.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
|
||||
std::vector<MemBlockType> neighbor_vec_blocks;
|
||||
int ret = entity.get_vector_typed(neighbor_ids.data(), size,
|
||||
neighbor_vec_blocks);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_vector())++;
|
||||
}
|
||||
@@ -364,14 +364,16 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
|
||||
break;
|
||||
}
|
||||
|
||||
static constexpr node_id_t PREFETCH_STEP = 2;
|
||||
std::vector<float> dists(size);
|
||||
std::vector<const void *> neighbor_vecs(size);
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
|
||||
}
|
||||
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
node_id_t node = neighbor_ids[i];
|
||||
node_id_t prefetch_id = i + PREFETCH_STEP;
|
||||
if (prefetch_id < size) {
|
||||
ailego_prefetch(neighbor_vec_blocks[prefetch_id].data());
|
||||
}
|
||||
dist_t cur_dist = dc.dist(neighbor_vec_blocks[i].data());
|
||||
dist_t cur_dist = dists[i];
|
||||
|
||||
if (!filter(node)) {
|
||||
std::string group_id = group_by(node);
|
||||
@@ -393,8 +395,10 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
|
||||
} // end if
|
||||
}
|
||||
|
||||
void HnswAlgorithm::update_neighbors(HnswDistCalculator &dc, node_id_t id,
|
||||
level_t level, TopkHeap &topk_heap) {
|
||||
template <typename EntityType>
|
||||
void HnswAlgorithm<EntityType>::update_neighbors(HnswDistCalculator &dc,
|
||||
node_id_t id, level_t level,
|
||||
TopkHeap &topk_heap) {
|
||||
topk_heap.sort();
|
||||
|
||||
uint32_t max_neighbor_cnt = entity_.neighbor_cnt(level);
|
||||
@@ -455,10 +459,10 @@ void HnswAlgorithm::update_neighbors(HnswDistCalculator &dc, node_id_t id,
|
||||
return;
|
||||
}
|
||||
|
||||
void HnswAlgorithm::reverse_update_neighbors(HnswDistCalculator &dc,
|
||||
node_id_t id, level_t level,
|
||||
node_id_t link_id, dist_t dist,
|
||||
TopkHeap &update_heap) {
|
||||
template <typename EntityType>
|
||||
void HnswAlgorithm<EntityType>::reverse_update_neighbors(
|
||||
HnswDistCalculator &dc, node_id_t id, level_t level, node_id_t link_id,
|
||||
dist_t dist, TopkHeap &update_heap) {
|
||||
const size_t max_neighbor_cnt = entity_.neighbor_cnt(level);
|
||||
|
||||
uint32_t lock_idx = id & kLockMask;
|
||||
@@ -516,5 +520,10 @@ void HnswAlgorithm::reverse_update_neighbors(HnswDistCalculator &dc,
|
||||
return;
|
||||
}
|
||||
|
||||
// Explicit template instantiation
|
||||
template class HnswAlgorithm<HnswMmapStreamerEntity>;
|
||||
template class HnswAlgorithm<HnswBufferPoolStreamerEntity>;
|
||||
template class HnswAlgorithm<HnswContiguousStreamerEntity>;
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
|
||||
@@ -14,41 +14,64 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <chrono>
|
||||
#include <vector>
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include "hnsw_context.h"
|
||||
#include "hnsw_dist_calculator.h"
|
||||
#include "hnsw_entity.h"
|
||||
#include "hnsw_streamer_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! hnsw graph algorithm implement
|
||||
class HnswAlgorithm {
|
||||
//! Non-template base class for HnswAlgorithm
|
||||
class HnswAlgorithmBase {
|
||||
public:
|
||||
typedef std::unique_ptr<HnswAlgorithm> UPointer;
|
||||
typedef std::unique_ptr<HnswAlgorithmBase> UPointer;
|
||||
|
||||
virtual ~HnswAlgorithmBase() = default;
|
||||
|
||||
virtual int cleanup() = 0;
|
||||
virtual int add_node(node_id_t id, level_t level, HnswContext *ctx) = 0;
|
||||
virtual int search(HnswContext *ctx) const = 0;
|
||||
virtual int init() = 0;
|
||||
virtual uint32_t get_random_level() const = 0;
|
||||
};
|
||||
|
||||
//! hnsw graph algorithm implement, templated on EntityType
|
||||
template <typename EntityType>
|
||||
class HnswAlgorithm : public HnswAlgorithmBase {
|
||||
public:
|
||||
using MemBlockType = typename EntityType::MemoryBlock;
|
||||
|
||||
//! Constructor
|
||||
explicit HnswAlgorithm(HnswEntity &entity);
|
||||
explicit HnswAlgorithm(EntityType &entity)
|
||||
: entity_(entity),
|
||||
mt_(std::chrono::system_clock::now().time_since_epoch().count()),
|
||||
lock_pool_(kLockCnt) {}
|
||||
|
||||
//! Destructor
|
||||
~HnswAlgorithm() = default;
|
||||
~HnswAlgorithm() override = default;
|
||||
|
||||
//! Cleanup HnswAlgorithm
|
||||
int cleanup();
|
||||
int cleanup() override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Add a node to hnsw graph
|
||||
//! @id: the node unique id
|
||||
//! @level: a node will be add to graph in each level [0, level]
|
||||
//! return 0 on success, or errCode in failure
|
||||
int add_node(node_id_t id, level_t level, HnswContext *ctx);
|
||||
int add_node(node_id_t id, level_t level, HnswContext *ctx) override;
|
||||
|
||||
//! do knn search in graph
|
||||
//! return 0 on success, or errCode in failure. results saved in ctx
|
||||
int search(HnswContext *ctx) const;
|
||||
int search(HnswContext *ctx) const override;
|
||||
|
||||
//! Initiate HnswAlgorithm
|
||||
int init() {
|
||||
int init() override {
|
||||
level_probas_.clear();
|
||||
double level_mult =
|
||||
1 / std::log(static_cast<double>(entity_.scaling_factor()));
|
||||
@@ -67,7 +90,7 @@ class HnswAlgorithm {
|
||||
|
||||
//! Generate a random level
|
||||
//! return graph level
|
||||
uint32_t get_random_level() const {
|
||||
uint32_t get_random_level() const override {
|
||||
// gen rand float (0, 1)
|
||||
double f = mt_() / static_cast<float>(mt_.max());
|
||||
for (size_t level = 0; level < level_probas_.size(); level++) {
|
||||
@@ -116,7 +139,7 @@ class HnswAlgorithm {
|
||||
static constexpr uint32_t kLockCnt{1U << 8};
|
||||
static constexpr uint32_t kLockMask{kLockCnt - 1U};
|
||||
|
||||
HnswEntity &entity_;
|
||||
EntityType &entity_;
|
||||
mutable std::mt19937 mt_{};
|
||||
std::vector<double> level_probas_{};
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ class ChunkBroker {
|
||||
CHUNK_TYPE_UPPER_NEIGHBOR = 4,
|
||||
CHUNK_TYPE_NEIGHBOR_INDEX = 5,
|
||||
CHUNK_TYPE_SPARSE_NODE = 6,
|
||||
CHUNK_TYPE_NEIGHBOR_DIST = 7, // Vamana: per-node neighbor distances
|
||||
CHUNK_TYPE_MAX = 8
|
||||
};
|
||||
static constexpr size_t kDefaultChunkSeqId = 0UL;
|
||||
|
||||
@@ -171,6 +171,142 @@ struct Neighbors {
|
||||
IndexStorage::MemoryBlock neighbor_block;
|
||||
};
|
||||
|
||||
//! Lightweight MemoryBlock for mmap mode: zero-cost construction/destruction
|
||||
struct MmapMemoryBlock {
|
||||
MmapMemoryBlock() = default;
|
||||
explicit MmapMemoryBlock(void *data) : data_(data) {}
|
||||
|
||||
MmapMemoryBlock(const MmapMemoryBlock &) = default;
|
||||
MmapMemoryBlock &operator=(const MmapMemoryBlock &) = default;
|
||||
MmapMemoryBlock(MmapMemoryBlock &&) = default;
|
||||
MmapMemoryBlock &operator=(MmapMemoryBlock &&) = default;
|
||||
~MmapMemoryBlock() = default;
|
||||
|
||||
const void *data() const {
|
||||
return data_;
|
||||
}
|
||||
|
||||
void reset(void *data) {
|
||||
data_ = data;
|
||||
}
|
||||
|
||||
void *data_{nullptr};
|
||||
};
|
||||
|
||||
//! Lightweight MemoryBlock for buffer pool mode: release on destruction
|
||||
struct BufferPoolMemoryBlock {
|
||||
BufferPoolMemoryBlock() = default;
|
||||
|
||||
BufferPoolMemoryBlock(ailego::VecBufferPoolHandle *handle, size_t block_id,
|
||||
void *data)
|
||||
: buffer_pool_handle_(handle), buffer_block_id_(block_id), data_(data) {}
|
||||
|
||||
BufferPoolMemoryBlock(const BufferPoolMemoryBlock &rhs)
|
||||
: buffer_pool_handle_(rhs.buffer_pool_handle_),
|
||||
buffer_block_id_(rhs.buffer_block_id_),
|
||||
data_(rhs.data_) {
|
||||
if (buffer_pool_handle_) {
|
||||
buffer_pool_handle_->acquire_one(buffer_block_id_);
|
||||
}
|
||||
}
|
||||
|
||||
BufferPoolMemoryBlock &operator=(const BufferPoolMemoryBlock &rhs) {
|
||||
if (this != &rhs) {
|
||||
release();
|
||||
buffer_pool_handle_ = rhs.buffer_pool_handle_;
|
||||
buffer_block_id_ = rhs.buffer_block_id_;
|
||||
data_ = rhs.data_;
|
||||
if (buffer_pool_handle_) {
|
||||
buffer_pool_handle_->acquire_one(buffer_block_id_);
|
||||
}
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
BufferPoolMemoryBlock(BufferPoolMemoryBlock &&rhs) noexcept
|
||||
: buffer_pool_handle_(rhs.buffer_pool_handle_),
|
||||
buffer_block_id_(rhs.buffer_block_id_),
|
||||
data_(rhs.data_) {
|
||||
rhs.buffer_pool_handle_ = nullptr;
|
||||
rhs.data_ = nullptr;
|
||||
}
|
||||
|
||||
BufferPoolMemoryBlock &operator=(BufferPoolMemoryBlock &&rhs) noexcept {
|
||||
if (this != &rhs) {
|
||||
release();
|
||||
buffer_pool_handle_ = rhs.buffer_pool_handle_;
|
||||
buffer_block_id_ = rhs.buffer_block_id_;
|
||||
data_ = rhs.data_;
|
||||
rhs.buffer_pool_handle_ = nullptr;
|
||||
rhs.data_ = nullptr;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
~BufferPoolMemoryBlock() {
|
||||
release();
|
||||
}
|
||||
|
||||
const void *data() const {
|
||||
return data_;
|
||||
}
|
||||
|
||||
void reset(ailego::VecBufferPoolHandle *handle, size_t block_id, void *data) {
|
||||
release();
|
||||
buffer_pool_handle_ = handle;
|
||||
buffer_block_id_ = block_id;
|
||||
data_ = data;
|
||||
}
|
||||
|
||||
private:
|
||||
void release() {
|
||||
if (buffer_pool_handle_) {
|
||||
buffer_pool_handle_->release_one(buffer_block_id_);
|
||||
buffer_pool_handle_ = nullptr;
|
||||
}
|
||||
data_ = nullptr;
|
||||
}
|
||||
|
||||
ailego::VecBufferPoolHandle *buffer_pool_handle_{nullptr};
|
||||
size_t buffer_block_id_{0};
|
||||
void *data_{nullptr};
|
||||
};
|
||||
|
||||
//! Typed Neighbors: holds a typed MemoryBlock to avoid runtime branching
|
||||
template <typename MemBlockType>
|
||||
struct NeighborsT {
|
||||
NeighborsT() : cnt{0}, data{nullptr} {}
|
||||
|
||||
NeighborsT(uint32_t cnt_in, const node_id_t *data_in)
|
||||
: cnt{cnt_in}, data{data_in} {}
|
||||
|
||||
explicit NeighborsT(const MemBlockType &mem_block)
|
||||
: neighbor_block{mem_block} {
|
||||
auto hd = reinterpret_cast<const NeighborsHeader *>(neighbor_block.data());
|
||||
cnt = hd->neighbor_cnt;
|
||||
data = hd->neighbors;
|
||||
}
|
||||
|
||||
explicit NeighborsT(MemBlockType &&mem_block)
|
||||
: neighbor_block{std::move(mem_block)} {
|
||||
auto hd = reinterpret_cast<const NeighborsHeader *>(neighbor_block.data());
|
||||
cnt = hd->neighbor_cnt;
|
||||
data = hd->neighbors;
|
||||
}
|
||||
|
||||
size_t size(void) const {
|
||||
return cnt;
|
||||
}
|
||||
|
||||
const node_id_t &operator[](size_t idx) const {
|
||||
return data[idx];
|
||||
}
|
||||
|
||||
uint32_t cnt;
|
||||
const node_id_t *data;
|
||||
MemBlockType neighbor_block;
|
||||
};
|
||||
|
||||
//! level 0 neighbors offset
|
||||
struct GraphNeighborMeta {
|
||||
GraphNeighborMeta(size_t o, size_t cnt) : offset(o), neighbor_cnt(cnt) {}
|
||||
|
||||
@@ -108,5 +108,8 @@ static const std::string PARAM_HNSW_REDUCER_INDEX_NAME(
|
||||
static const std::string PARAM_HNSW_REDUCER_EFCONSTRUCTION(
|
||||
"proxima.hnsw.reducer.efconstruction");
|
||||
|
||||
static const std::string PARAM_HNSW_STREAMER_USE_CONTIGUOUS_MEMORY(
|
||||
"proxima.hnsw.streamer.use_contiguous_memory");
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
HnswStreamer::HnswStreamer() : entity_(stats_) {}
|
||||
HnswStreamer::HnswStreamer() = default;
|
||||
|
||||
HnswStreamer::~HnswStreamer() {
|
||||
if (state_ == STATE_INITED) {
|
||||
@@ -68,7 +68,8 @@ int HnswStreamer::init(const IndexMeta &imeta, const ailego::Params ¶ms) {
|
||||
params.get(PARAM_HNSW_STREAMER_FORCE_PADDING_RESULT_ENABLE,
|
||||
&force_padding_topk_enabled_);
|
||||
params.get(PARAM_HNSW_STREAMER_USE_ID_MAP, &use_id_map_);
|
||||
entity_.set_use_key_info_map(use_id_map_);
|
||||
params.get(PARAM_HNSW_STREAMER_USE_CONTIGUOUS_MEMORY,
|
||||
&use_contiguous_memory_);
|
||||
|
||||
params.get(PARAM_HNSW_STREAMER_DOCS_SOFT_LIMIT, &docs_soft_limit_);
|
||||
if (docs_soft_limit_ > 0 && docs_soft_limit_ > docs_hard_limit_) {
|
||||
@@ -143,6 +144,7 @@ int HnswStreamer::init(const IndexMeta &imeta, const ailego::Params ¶ms) {
|
||||
if (prune_cnt == 0UL) {
|
||||
prune_cnt = upper_max_neighbor_cnt_;
|
||||
}
|
||||
prune_cnt_ = prune_cnt;
|
||||
if (chunk_size_ == 0UL) {
|
||||
chunk_size_ = HnswEntity::kDefaultChunkSize;
|
||||
}
|
||||
@@ -152,47 +154,21 @@ int HnswStreamer::init(const IndexMeta &imeta, const ailego::Params ¶ms) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
entity_.set_ef_construction(ef_construction_);
|
||||
entity_.set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
|
||||
entity_.set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
|
||||
entity_.set_scaling_factor(scaling_factor_);
|
||||
entity_.set_prune_cnt(prune_cnt);
|
||||
|
||||
entity_.set_vector_size(meta_.element_size());
|
||||
|
||||
entity_.set_chunk_size(chunk_size_);
|
||||
entity_.set_filter_same_key(filter_same_key_);
|
||||
entity_.set_get_vector(get_vector_enabled_);
|
||||
entity_.set_min_neighbor_cnt(min_neighbor_cnt_);
|
||||
|
||||
int ret = entity_.init(docs_hard_limit_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Hnsw entity init failed for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
|
||||
LOG_DEBUG(
|
||||
"Init params: maxIndexSize=%zu docsHardLimit=%zu docsSoftLimit=%zu "
|
||||
"efConstruction=%u ef=%u upperMaxNeighborCnt=%u l0MaxNeighborCnt=%u "
|
||||
"scalingFactor=%u maxScanRatio=%.3f minScanLimit=%zu maxScanLimit=%zu "
|
||||
"bfEnabled=%d bruteFoceThreshold=%zu bfNegativeProbability=%.5f "
|
||||
"checkCrcEnabled=%d pruneSize=%zu vectorSize=%u chunkSize=%zu "
|
||||
"checkCrcEnabled=%d pruneSize=%u vectorSize=%u chunkSize=%zu "
|
||||
"filterSameKey=%u getVectorEnabled=%u minNeighborCount=%u "
|
||||
"forcePadding=%u ",
|
||||
max_index_size_, docs_hard_limit_, docs_soft_limit_, ef_construction_,
|
||||
ef_, upper_max_neighbor_cnt_, l0_max_neighbor_cnt_, scaling_factor_,
|
||||
max_scan_ratio_, min_scan_limit_, max_scan_limit_, bf_enabled_,
|
||||
bruteforce_threshold_, bf_negative_prob_, check_crc_enabled_, prune_cnt,
|
||||
bruteforce_threshold_, bf_negative_prob_, check_crc_enabled_, prune_cnt_,
|
||||
meta_.element_size(), chunk_size_, filter_same_key_, get_vector_enabled_,
|
||||
min_neighbor_cnt_, force_padding_topk_enabled_);
|
||||
|
||||
alg_ = HnswAlgorithm::UPointer(new HnswAlgorithm(entity_));
|
||||
|
||||
ret = alg_->init();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
@@ -208,7 +184,9 @@ int HnswStreamer::cleanup(void) {
|
||||
meta_.clear();
|
||||
metric_.reset();
|
||||
stats_.clear();
|
||||
entity_.cleanup();
|
||||
if (entity_) {
|
||||
entity_->cleanup();
|
||||
}
|
||||
|
||||
if (alg_) {
|
||||
alg_->cleanup();
|
||||
@@ -237,6 +215,26 @@ int HnswStreamer::cleanup(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HnswStreamer::setup_entity() {
|
||||
entity_->set_use_key_info_map(use_id_map_);
|
||||
entity_->set_ef_construction(ef_construction_);
|
||||
entity_->set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
|
||||
entity_->set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
|
||||
entity_->set_scaling_factor(scaling_factor_);
|
||||
entity_->set_prune_cnt(prune_cnt_);
|
||||
entity_->set_vector_size(meta_.element_size());
|
||||
entity_->set_chunk_size(chunk_size_);
|
||||
entity_->set_filter_same_key(filter_same_key_);
|
||||
entity_->set_get_vector(get_vector_enabled_);
|
||||
entity_->set_min_neighbor_cnt(min_neighbor_cnt_);
|
||||
|
||||
int ret = entity_->init(docs_hard_limit_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Hnsw entity init failed for %s", IndexError::What(ret));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int HnswStreamer::open(IndexStorage::Pointer stg) {
|
||||
LOG_INFO("HnswStreamer open");
|
||||
|
||||
@@ -244,15 +242,36 @@ int HnswStreamer::open(IndexStorage::Pointer stg) {
|
||||
LOG_ERROR("Open storage failed, init streamer first!");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
int ret = entity_.open(std::move(stg), max_index_size_, check_crc_enabled_);
|
||||
|
||||
// Create entity based on storage type
|
||||
switch (stg->memory_block_type()) {
|
||||
case IndexStorage::MemoryBlock::MBT_BUFFERPOOL: {
|
||||
entity_ = std::make_unique<HnswBufferPoolStreamerEntity>(stats_);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
if (use_contiguous_memory_) {
|
||||
entity_ = std::make_unique<HnswContiguousStreamerEntity>(stats_);
|
||||
} else {
|
||||
entity_ = std::make_unique<HnswMmapStreamerEntity>(stats_);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
int ret = setup_entity();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = entity_->open(std::move(stg), max_index_size_, check_crc_enabled_);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
IndexMeta index_meta;
|
||||
ret = entity_.get_index_meta(&index_meta);
|
||||
ret = entity_->get_index_meta(&index_meta);
|
||||
if (ret == IndexError_NoExist) {
|
||||
// Set IndexMeta for the new index
|
||||
ret = entity_.set_index_meta(meta_);
|
||||
ret = entity_->set_index_meta(meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to set index meta for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
@@ -307,6 +326,36 @@ int HnswStreamer::open(IndexStorage::Pointer stg) {
|
||||
search_batch_distance_ = metric_->query_metric()->batch_distance();
|
||||
}
|
||||
|
||||
// Create algorithm based on entity storage mode
|
||||
switch (entity_->storage_mode()) {
|
||||
case HnswStorageMode::kBufferPool:
|
||||
alg_ = HnswAlgorithmBase::UPointer(
|
||||
new HnswAlgorithm<HnswBufferPoolStreamerEntity>(
|
||||
static_cast<HnswBufferPoolStreamerEntity &>(*entity_)));
|
||||
break;
|
||||
case HnswStorageMode::kContiguous: {
|
||||
auto &contiguous_entity =
|
||||
static_cast<HnswContiguousStreamerEntity &>(*entity_);
|
||||
int build_ret = contiguous_entity.build_contiguous_memory();
|
||||
if (build_ret != 0) {
|
||||
LOG_ERROR("Failed to build contiguous memory, ret=%d", build_ret);
|
||||
return build_ret;
|
||||
}
|
||||
alg_ = HnswAlgorithmBase::UPointer(
|
||||
new HnswAlgorithm<HnswContiguousStreamerEntity>(contiguous_entity));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
alg_ =
|
||||
HnswAlgorithmBase::UPointer(new HnswAlgorithm<HnswMmapStreamerEntity>(
|
||||
static_cast<HnswMmapStreamerEntity &>(*entity_)));
|
||||
break;
|
||||
}
|
||||
ret = alg_->init();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
state_ = STATE_OPENED;
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
@@ -318,8 +367,8 @@ int HnswStreamer::close(void) {
|
||||
|
||||
stats_.clear();
|
||||
meta_.set_metric(metric_->name(), 0, metric_->params());
|
||||
entity_.set_index_meta(meta_);
|
||||
int ret = entity_.close();
|
||||
entity_->set_index_meta(meta_);
|
||||
int ret = entity_->close();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
@@ -332,8 +381,8 @@ int HnswStreamer::flush(uint64_t checkpoint) {
|
||||
LOG_INFO("HnswStreamer flush checkpoint=%zu", (size_t)checkpoint);
|
||||
|
||||
meta_.set_metric(metric_->name(), 0, metric_->params());
|
||||
entity_.set_index_meta(meta_);
|
||||
return entity_.flush(checkpoint);
|
||||
entity_->set_index_meta(meta_);
|
||||
return entity_->flush(checkpoint);
|
||||
}
|
||||
|
||||
int HnswStreamer::dump(const IndexDumper::Pointer &dumper) {
|
||||
@@ -347,7 +396,7 @@ int HnswStreamer::dump(const IndexDumper::Pointer &dumper) {
|
||||
LOG_ERROR("Failed to serialize meta into dumper.");
|
||||
return ret;
|
||||
}
|
||||
return entity_.dump(dumper);
|
||||
return entity_->dump(dumper);
|
||||
}
|
||||
|
||||
IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
|
||||
@@ -356,7 +405,7 @@ IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
HnswEntity::Pointer entity = entity_.clone();
|
||||
HnswEntity::Pointer entity = entity_->clone();
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("CreateContext clone init failed");
|
||||
return Context::Pointer();
|
||||
@@ -387,9 +436,9 @@ IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
|
||||
if (meta_.streamer_params().get(PARAM_HNSW_STREAMER_ESTIMATE_DOC_COUNT,
|
||||
&estimate_doc_count)) {
|
||||
LOG_DEBUG("HnswStreamer doc_count[%zu] estimate[%zu]",
|
||||
(size_t)entity_.doc_cnt(), (size_t)estimate_doc_count);
|
||||
(size_t)entity_->doc_cnt(), (size_t)estimate_doc_count);
|
||||
}
|
||||
ctx->check_need_adjuct_ctx(std::max(entity_.doc_cnt(), estimate_doc_count));
|
||||
ctx->check_need_adjuct_ctx(std::max(entity_->doc_cnt(), estimate_doc_count));
|
||||
|
||||
return Context::Pointer(ctx);
|
||||
}
|
||||
@@ -397,7 +446,7 @@ IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
|
||||
IndexProvider::Pointer HnswStreamer::create_provider(void) const {
|
||||
LOG_DEBUG("HnswStreamer create provider");
|
||||
|
||||
auto entity = entity_.clone();
|
||||
auto entity = entity_->clone();
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("Clone HnswEntity failed");
|
||||
return nullptr;
|
||||
@@ -407,7 +456,7 @@ IndexProvider::Pointer HnswStreamer::create_provider(void) const {
|
||||
}
|
||||
|
||||
int HnswStreamer::update_context(HnswContext *ctx) const {
|
||||
const HnswEntity::Pointer entity = entity_.clone();
|
||||
const HnswEntity::Pointer entity = entity_->clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone search context entity");
|
||||
return IndexError_Runtime;
|
||||
@@ -442,15 +491,15 @@ int HnswStreamer::add_with_id_impl(uint32_t id, const void *query,
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
|
||||
if (entity_.doc_cnt() >= docs_hard_limit_) {
|
||||
LOG_ERROR("Current docs %u exceed [%s]", entity_.doc_cnt(),
|
||||
if (ailego_unlikely(entity_->doc_cnt() >= docs_soft_limit_)) {
|
||||
if (entity_->doc_cnt() >= docs_hard_limit_) {
|
||||
LOG_ERROR("Current docs %u exceed [%s]", entity_->doc_cnt(),
|
||||
PARAM_HNSW_STREAMER_DOCS_HARD_LIMIT.c_str());
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_IndexFull;
|
||||
} else {
|
||||
LOG_WARN("Current docs %u exceed [%s]", entity_.doc_cnt(),
|
||||
LOG_WARN("Current docs %u exceed [%s]", entity_->doc_cnt(),
|
||||
PARAM_HNSW_STREAMER_DOCS_SOFT_LIMIT.c_str());
|
||||
}
|
||||
}
|
||||
@@ -464,7 +513,7 @@ int HnswStreamer::add_with_id_impl(uint32_t id, const void *query,
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
|
||||
ctx->reset_query(query);
|
||||
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
|
||||
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
|
||||
|
||||
if (metric_->support_train()) {
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
@@ -477,7 +526,7 @@ int HnswStreamer::add_with_id_impl(uint32_t id, const void *query,
|
||||
}
|
||||
|
||||
level_t level = alg_->get_random_level();
|
||||
ret = entity_.add_vector_with_id(level, id, query);
|
||||
ret = entity_->add_vector_with_id(level, id, query);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw streamer add vector failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
@@ -522,15 +571,15 @@ int HnswStreamer::add_impl(uint64_t pkey, const void *query,
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(entity_.doc_cnt() >= docs_soft_limit_)) {
|
||||
if (entity_.doc_cnt() >= docs_hard_limit_) {
|
||||
LOG_ERROR("Current docs %u exceed [%s]", entity_.doc_cnt(),
|
||||
if (ailego_unlikely(entity_->doc_cnt() >= docs_soft_limit_)) {
|
||||
if (entity_->doc_cnt() >= docs_hard_limit_) {
|
||||
LOG_ERROR("Current docs %u exceed [%s]", entity_->doc_cnt(),
|
||||
PARAM_HNSW_STREAMER_DOCS_HARD_LIMIT.c_str());
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_IndexFull;
|
||||
} else {
|
||||
LOG_WARN("Current docs %u exceed [%s]", entity_.doc_cnt(),
|
||||
LOG_WARN("Current docs %u exceed [%s]", entity_->doc_cnt(),
|
||||
PARAM_HNSW_STREAMER_DOCS_SOFT_LIMIT.c_str());
|
||||
}
|
||||
}
|
||||
@@ -544,7 +593,7 @@ int HnswStreamer::add_impl(uint64_t pkey, const void *query,
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
|
||||
ctx->reset_query(query);
|
||||
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
|
||||
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
|
||||
|
||||
if (metric_->support_train()) {
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
@@ -558,7 +607,7 @@ int HnswStreamer::add_impl(uint64_t pkey, const void *query,
|
||||
|
||||
level_t level = alg_->get_random_level();
|
||||
node_id_t id;
|
||||
ret = entity_.add_vector(level, pkey, query, &id);
|
||||
ret = entity_->add_vector(level, pkey, query, &id);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Hnsw streamer add vector failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
@@ -601,7 +650,7 @@ int HnswStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (entity_.doc_cnt() <= ctx->get_bruteforce_threshold()) {
|
||||
if (entity_->doc_cnt() <= ctx->get_bruteforce_threshold()) {
|
||||
return search_bf_impl(query, qmeta, count, context);
|
||||
}
|
||||
|
||||
@@ -616,7 +665,7 @@ int HnswStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
|
||||
ctx->resize_results(count);
|
||||
ctx->check_need_adjuct_ctx(entity_.doc_cnt());
|
||||
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
ctx->reset_query(query);
|
||||
ret = alg_->search(ctx);
|
||||
@@ -636,11 +685,11 @@ int HnswStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
}
|
||||
|
||||
void HnswStreamer::print_debug_info() {
|
||||
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
|
||||
if (entity_.get_key(id) == kInvalidKey) {
|
||||
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
|
||||
if (entity_->get_key(id) == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
Neighbors neighbours = entity_.get_neighbors(0, id);
|
||||
Neighbors neighbours = entity_->get_neighbors(0, id);
|
||||
std::cout << "node: " << id << "; ";
|
||||
if (neighbours.size() == 0) std::cout << std::endl;
|
||||
for (uint32_t i = 0; i < neighbours.size(); ++i) {
|
||||
@@ -654,7 +703,7 @@ void HnswStreamer::print_debug_info() {
|
||||
}
|
||||
}
|
||||
|
||||
// entity_.print_key_map();
|
||||
// entity_->print_key_map();
|
||||
}
|
||||
|
||||
int HnswStreamer::search_bf_impl(
|
||||
@@ -694,19 +743,19 @@ int HnswStreamer::search_bf_impl(
|
||||
}
|
||||
|
||||
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
|
||||
return ctx->group_by()(entity_.get_key(id));
|
||||
return ctx->group_by()(entity_->get_key(id));
|
||||
};
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
ctx->reset_query(query);
|
||||
ctx->group_topk_heaps().clear();
|
||||
|
||||
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
|
||||
if (entity_.get_key(id) == kInvalidKey) {
|
||||
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
|
||||
if (entity_->get_key(id) == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(entity_.get_key(id))) {
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(entity_->get_key(id))) {
|
||||
dist_t dist = ctx->dist_calculator().batch_dist(id);
|
||||
|
||||
std::string group_id = group_by(id);
|
||||
@@ -728,12 +777,12 @@ int HnswStreamer::search_bf_impl(
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
ctx->reset_query(query);
|
||||
topk.clear();
|
||||
for (node_id_t id = 0; id < entity_.doc_cnt(); ++id) {
|
||||
if (entity_.get_key(id) == kInvalidKey) {
|
||||
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
|
||||
if (entity_->get_key(id) == kInvalidKey) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!filter.is_valid() || !filter(entity_.get_key(id))) {
|
||||
if (!filter.is_valid() || !filter(entity_->get_key(id))) {
|
||||
dist_t dist = ctx->dist_calculator().batch_dist(id);
|
||||
topk.emplace(id, dist);
|
||||
}
|
||||
@@ -788,7 +837,7 @@ int HnswStreamer::search_bf_by_p_keys_impl(
|
||||
}
|
||||
|
||||
std::function<std::string(node_id_t)> group_by = [&](node_id_t id) {
|
||||
return ctx->group_by()(entity_.get_key(id));
|
||||
return ctx->group_by()(entity_->get_key(id));
|
||||
};
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
@@ -798,7 +847,7 @@ int HnswStreamer::search_bf_by_p_keys_impl(
|
||||
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
|
||||
uint64_t pk = p_keys[q][idx];
|
||||
if (!ctx->filter().is_valid() || !ctx->filter()(pk)) {
|
||||
node_id_t id = entity_.get_id(pk);
|
||||
node_id_t id = entity_->get_id(pk);
|
||||
if (id != kInvalidNodeId) {
|
||||
dist_t dist = ctx->dist_calculator().batch_dist(id);
|
||||
std::string group_id = group_by(id);
|
||||
@@ -824,7 +873,7 @@ int HnswStreamer::search_bf_by_p_keys_impl(
|
||||
for (size_t idx = 0; idx < p_keys[q].size(); ++idx) {
|
||||
key_t pk = p_keys[q][idx];
|
||||
if (!filter.is_valid() || !filter(pk)) {
|
||||
node_id_t id = entity_.get_id(pk);
|
||||
node_id_t id = entity_->get_id(pk);
|
||||
if (id != kInvalidNodeId) {
|
||||
dist_t dist = ctx->dist_calculator().batch_dist(id);
|
||||
topk.emplace(id, dist);
|
||||
|
||||
@@ -31,6 +31,17 @@ class HnswStreamer : public IndexStreamer {
|
||||
HnswStreamer(const HnswStreamer &streamer) = delete;
|
||||
HnswStreamer &operator=(const HnswStreamer &streamer) = delete;
|
||||
|
||||
public:
|
||||
//! Retrieve the storage mode of the underlying entity. Returns
|
||||
//! HnswStorageMode::kMmap when the entity has not been initialized yet.
|
||||
//! Intended for introspection and debug/testing usage.
|
||||
HnswStorageMode storage_mode() const {
|
||||
if (!entity_) {
|
||||
return HnswStorageMode::kMmap;
|
||||
}
|
||||
return entity_->storage_mode();
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Initialize Streamer
|
||||
virtual int init(const IndexMeta &imeta,
|
||||
@@ -88,22 +99,22 @@ class HnswStreamer : public IndexStreamer {
|
||||
|
||||
//! Fetch vector by key
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
return entity_.get_vector_by_key(key);
|
||||
return entity_->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
virtual int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_.get_vector_by_key(key, block);
|
||||
return entity_->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
//! Fetch vector by id
|
||||
virtual const void *get_vector_by_id(uint32_t id) const override {
|
||||
return entity_.get_vector(id);
|
||||
return entity_->get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_id(
|
||||
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_.get_vector(id, block);
|
||||
return entity_->get_vector(id, block);
|
||||
}
|
||||
|
||||
//! Open index from file path
|
||||
@@ -159,6 +170,9 @@ class HnswStreamer : public IndexStreamer {
|
||||
}
|
||||
|
||||
private:
|
||||
//! Configure and initialize the entity with saved parameters
|
||||
int setup_entity();
|
||||
|
||||
//! To share ctx across streamer/searcher, we need to update the context for
|
||||
//! current streamer/searcher
|
||||
int update_context(HnswContext *ctx) const;
|
||||
@@ -181,8 +195,8 @@ class HnswStreamer : public IndexStreamer {
|
||||
}
|
||||
};
|
||||
|
||||
HnswStreamerEntity entity_;
|
||||
HnswAlgorithm::UPointer alg_;
|
||||
std::unique_ptr<HnswStreamerEntity> entity_;
|
||||
HnswAlgorithmBase::UPointer alg_;
|
||||
IndexMeta meta_{};
|
||||
IndexMetric::Pointer metric_{};
|
||||
|
||||
@@ -200,6 +214,7 @@ class HnswStreamer : public IndexStreamer {
|
||||
size_t docs_hard_limit_{HnswEntity::kDefaultDocsHardLimit};
|
||||
size_t docs_soft_limit_{0UL};
|
||||
uint32_t min_neighbor_cnt_{0u};
|
||||
uint32_t prune_cnt_{0u};
|
||||
uint32_t upper_max_neighbor_cnt_{HnswEntity::kDefaultUpperMaxNeighborCnt};
|
||||
uint32_t l0_max_neighbor_cnt_{HnswEntity::kDefaultL0MaxNeighborCnt};
|
||||
uint32_t ef_{HnswEntity::kDefaultEf};
|
||||
@@ -219,6 +234,7 @@ class HnswStreamer : public IndexStreamer {
|
||||
bool get_vector_enabled_{false};
|
||||
bool force_padding_topk_enabled_{false};
|
||||
bool use_id_map_{true};
|
||||
bool use_contiguous_memory_{false};
|
||||
|
||||
//! avoid add vector while dumping index
|
||||
ailego::SharedMutex shared_mutex_{};
|
||||
|
||||
@@ -13,6 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
#include "hnsw_streamer_entity.h"
|
||||
#if defined(__linux__) || defined(__APPLE__)
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#include <ailego/utility/memory_helper.h>
|
||||
|
||||
// #define DEBUG_PRINT
|
||||
@@ -773,5 +776,214 @@ const HnswEntity::Pointer HnswStreamerEntity::clone() const {
|
||||
return HnswEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
const HnswEntity::Pointer HnswMmapStreamerEntity::clone() const {
|
||||
std::vector<Chunk::Pointer> node_chunks;
|
||||
node_chunks.reserve(node_chunks_.size());
|
||||
for (size_t i = 0UL; i < node_chunks_.size(); ++i) {
|
||||
node_chunks.emplace_back(node_chunks_[i]->clone());
|
||||
if (ailego_unlikely(!node_chunks[i])) {
|
||||
LOG_ERROR("HnswMmapStreamerEntity get chunk failed in clone");
|
||||
return HnswEntity::Pointer();
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Chunk::Pointer> upper_neighbor_chunks;
|
||||
upper_neighbor_chunks.reserve(upper_neighbor_chunks_.size());
|
||||
for (size_t i = 0UL; i < upper_neighbor_chunks_.size(); ++i) {
|
||||
upper_neighbor_chunks.emplace_back(upper_neighbor_chunks_[i]->clone());
|
||||
if (ailego_unlikely(!upper_neighbor_chunks[i])) {
|
||||
LOG_ERROR("HnswMmapStreamerEntity get chunk failed in clone");
|
||||
return HnswEntity::Pointer();
|
||||
}
|
||||
}
|
||||
|
||||
auto *entity = new (std::nothrow) HnswMmapStreamerEntity(
|
||||
stats_, header(), chunk_size_, node_index_mask_bits_,
|
||||
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
|
||||
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
|
||||
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_,
|
||||
nullptr, nullptr);
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("HnswMmapStreamerEntity new failed");
|
||||
}
|
||||
return HnswEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
const HnswEntity::Pointer HnswContiguousStreamerEntity::clone() const {
|
||||
std::vector<Chunk::Pointer> node_chunks;
|
||||
node_chunks.reserve(node_chunks_.size());
|
||||
for (size_t i = 0UL; i < node_chunks_.size(); ++i) {
|
||||
node_chunks.emplace_back(node_chunks_[i]->clone());
|
||||
if (ailego_unlikely(!node_chunks[i])) {
|
||||
LOG_ERROR("HnswContiguousStreamerEntity get chunk failed in clone");
|
||||
return HnswEntity::Pointer();
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Chunk::Pointer> upper_neighbor_chunks;
|
||||
upper_neighbor_chunks.reserve(upper_neighbor_chunks_.size());
|
||||
for (size_t i = 0UL; i < upper_neighbor_chunks_.size(); ++i) {
|
||||
upper_neighbor_chunks.emplace_back(upper_neighbor_chunks_[i]->clone());
|
||||
if (ailego_unlikely(!upper_neighbor_chunks[i])) {
|
||||
LOG_ERROR("HnswContiguousStreamerEntity get chunk failed in clone");
|
||||
return HnswEntity::Pointer();
|
||||
}
|
||||
}
|
||||
|
||||
auto *entity = new (std::nothrow) HnswContiguousStreamerEntity(
|
||||
stats_, header(), chunk_size_, node_index_mask_bits_,
|
||||
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
|
||||
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
|
||||
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_,
|
||||
nullptr, nullptr);
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("HnswContiguousStreamerEntity new failed");
|
||||
return HnswEntity::Pointer();
|
||||
}
|
||||
|
||||
// Share contiguous memory with the clone (zero-copy)
|
||||
entity->node_memory_ = node_memory_;
|
||||
entity->node_base_ = node_base_;
|
||||
entity->upper_neighbor_memory_ = upper_neighbor_memory_;
|
||||
entity->upper_neighbor_base_ = upper_neighbor_base_;
|
||||
entity->upper_chunk_offsets_ = upper_chunk_offsets_;
|
||||
|
||||
return HnswEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// HnswContiguousStreamerEntity implementation
|
||||
// ============================================================================
|
||||
|
||||
char *HnswContiguousStreamerEntity::allocate_contiguous(size_t size) {
|
||||
if (size == 0) {
|
||||
return nullptr;
|
||||
}
|
||||
#if defined(__linux__)
|
||||
// Use mmap with MAP_ANONYMOUS for contiguous memory
|
||||
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (ptr == MAP_FAILED) {
|
||||
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
|
||||
return nullptr;
|
||||
}
|
||||
// Request transparent huge pages
|
||||
::madvise(ptr, size, MADV_HUGEPAGE);
|
||||
return static_cast<char *>(ptr);
|
||||
#elif defined(__APPLE__)
|
||||
// macOS: use mmap with MAP_ANONYMOUS
|
||||
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANON, -1, 0);
|
||||
if (ptr == MAP_FAILED) {
|
||||
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
|
||||
return nullptr;
|
||||
}
|
||||
return static_cast<char *>(ptr);
|
||||
#elif defined(_WIN32)
|
||||
void *ptr = ::_aligned_malloc(size, ailego::MemoryHelper::PageSize());
|
||||
if (!ptr) {
|
||||
LOG_ERROR("_aligned_malloc failed for contiguous memory, size=%zu", size);
|
||||
return nullptr;
|
||||
}
|
||||
return static_cast<char *>(ptr);
|
||||
#else
|
||||
// Fallback: aligned allocation
|
||||
void *ptr = std::aligned_alloc(ailego::MemoryHelper::PageSize(), size);
|
||||
if (!ptr) {
|
||||
LOG_ERROR("aligned_alloc failed for contiguous memory, size=%zu", size);
|
||||
return nullptr;
|
||||
}
|
||||
return static_cast<char *>(ptr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int HnswContiguousStreamerEntity::build_contiguous_memory() {
|
||||
node_memory_.reset();
|
||||
node_base_ = nullptr;
|
||||
upper_neighbor_memory_.reset();
|
||||
upper_neighbor_base_ = nullptr;
|
||||
upper_chunk_offsets_.clear();
|
||||
|
||||
const uint32_t total_docs = doc_cnt();
|
||||
if (total_docs == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// --- Build contiguous node memory ---
|
||||
const size_t per_node = node_size();
|
||||
const size_t total_node_data = static_cast<size_t>(total_docs) * per_node;
|
||||
size_t node_memory_size = AlignHugePageSize(total_node_data);
|
||||
char *raw_node = allocate_contiguous(node_memory_size);
|
||||
if (!raw_node) {
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
node_memory_.reset(raw_node, ContiguousDeleter{node_memory_size});
|
||||
node_base_ = raw_node;
|
||||
|
||||
// Copy node data from chunks into contiguous memory
|
||||
// Each chunk holds node_cnt_per_chunk nodes, laid out at offset
|
||||
// (id & mask) * node_size within the chunk.
|
||||
const auto &chunks = node_chunks_;
|
||||
const uint32_t nodes_per_chunk = 1U << node_index_mask_bits_;
|
||||
for (size_t chunk_idx = 0; chunk_idx < chunks.size(); ++chunk_idx) {
|
||||
const void *chunk_data = nullptr;
|
||||
size_t data_size = chunks[chunk_idx]->data_size();
|
||||
chunks[chunk_idx]->read(0, &chunk_data, data_size);
|
||||
|
||||
// Number of nodes in this chunk
|
||||
uint32_t base_id = chunk_idx * nodes_per_chunk;
|
||||
uint32_t count_in_chunk = std::min(nodes_per_chunk, total_docs - base_id);
|
||||
|
||||
// Copy each node's data
|
||||
const char *src = static_cast<const char *>(chunk_data);
|
||||
char *dst = node_base_ + static_cast<size_t>(base_id) * per_node;
|
||||
std::memcpy(dst, src, static_cast<size_t>(count_in_chunk) * per_node);
|
||||
}
|
||||
|
||||
// --- Build contiguous upper neighbor memory ---
|
||||
const auto &upper_chunks = upper_neighbor_chunks_;
|
||||
if (upper_chunks.empty()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Sync all upper neighbor chunks
|
||||
sync_upper_neighbor_chunks(upper_chunks.size() - 1);
|
||||
|
||||
// Calculate cumulative offsets and total size
|
||||
upper_chunk_offsets_.resize(upper_chunks.size());
|
||||
size_t total_upper_size = 0;
|
||||
for (size_t i = 0; i < upper_chunks.size(); ++i) {
|
||||
upper_chunk_offsets_[i] = total_upper_size;
|
||||
total_upper_size += upper_chunks[i]->data_size();
|
||||
}
|
||||
|
||||
size_t upper_memory_size = AlignHugePageSize(total_upper_size);
|
||||
char *raw_upper = allocate_contiguous(upper_memory_size);
|
||||
if (!raw_upper) {
|
||||
node_memory_.reset();
|
||||
node_base_ = nullptr;
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
upper_neighbor_memory_.reset(raw_upper, ContiguousDeleter{upper_memory_size});
|
||||
upper_neighbor_base_ = raw_upper;
|
||||
|
||||
// Copy upper neighbor data from chunks into contiguous memory
|
||||
for (size_t i = 0; i < upper_chunks.size(); ++i) {
|
||||
const void *chunk_data = nullptr;
|
||||
size_t data_size = upper_chunks[i]->data_size();
|
||||
upper_chunks[i]->read(0, &chunk_data, data_size);
|
||||
std::memcpy(upper_neighbor_base_ + upper_chunk_offsets_[i], chunk_data,
|
||||
data_size);
|
||||
}
|
||||
|
||||
LOG_INFO(
|
||||
"Built contiguous memory: node_size=%zu upper_neighbor_size=%zu "
|
||||
"total_docs=%u node_chunks=%zu upper_chunks=%zu",
|
||||
node_memory_size, upper_memory_size, total_docs, chunks.size(),
|
||||
upper_chunks.size());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
|
||||
@@ -15,6 +15,11 @@
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#if defined(__linux__) || defined(__APPLE__)
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <sparsehash/dense_hash_map>
|
||||
#include <sparsehash/dense_hash_set>
|
||||
@@ -28,6 +33,10 @@
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
|
||||
//! Storage mode for HnswStreamerEntity
|
||||
enum class HnswStorageMode { kMmap = 0, kBufferPool = 1, kContiguous = 2 };
|
||||
|
||||
//! HnswStreamerEntity manage vector data, pkey, and node's neighbors
|
||||
class HnswStreamerEntity : public HnswEntity {
|
||||
public:
|
||||
@@ -84,6 +93,11 @@ class HnswStreamerEntity : public HnswEntity {
|
||||
|
||||
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
|
||||
|
||||
//! Get the storage mode of this entity
|
||||
virtual HnswStorageMode storage_mode() const {
|
||||
return HnswStorageMode::kMmap;
|
||||
}
|
||||
|
||||
void set_use_key_info_map(bool use_id_map) {
|
||||
use_key_info_map_ = use_id_map;
|
||||
LOG_DEBUG("use_key_info_map_: %d", (int)use_key_info_map_);
|
||||
@@ -178,6 +192,22 @@ class HnswStreamerEntity : public HnswEntity {
|
||||
std::cout << "key map ends" << std::endl;
|
||||
}
|
||||
|
||||
//! Typed get_neighbors: returns NeighborsT<MemBlock> without runtime
|
||||
//! branching on MemoryBlock type. MmapMemoryBlock specialization uses
|
||||
//! pointer-based read; BufferPoolMemoryBlock uses MemoryBlock-based read.
|
||||
template <typename MemBlock>
|
||||
inline NeighborsT<MemBlock> get_neighbors_typed(level_t level,
|
||||
node_id_t id) const;
|
||||
|
||||
//! Typed batch get_vector: fills vector<MemBlock> without runtime branching
|
||||
template <typename MemBlock>
|
||||
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
|
||||
std::vector<MemBlock> &vec_blocks) const;
|
||||
|
||||
//! Typed get_key: reads key using typed MemBlock
|
||||
template <typename MemBlock>
|
||||
inline key_t get_key_typed(node_id_t id) const;
|
||||
|
||||
//! Get l0 neighbors size
|
||||
inline size_t neighbors_size() const {
|
||||
return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
|
||||
@@ -189,7 +219,7 @@ class HnswStreamerEntity : public HnswEntity {
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
protected:
|
||||
union UpperNeighborIndexMeta {
|
||||
struct {
|
||||
uint32_t level : 4;
|
||||
@@ -200,6 +230,7 @@ class HnswStreamerEntity : public HnswEntity {
|
||||
uint32_t data;
|
||||
};
|
||||
|
||||
protected:
|
||||
template <class Key, class T>
|
||||
using HashMap = google::dense_hash_map<Key, T, std::hash<Key>>;
|
||||
template <class Key, class T>
|
||||
@@ -214,7 +245,7 @@ class HnswStreamerEntity : public HnswEntity {
|
||||
using NIHashMap = HnswIndexHashMap<node_id_t, uint32_t>;
|
||||
using NIHashMapPointer = std::shared_ptr<NIHashMap>;
|
||||
|
||||
//! Private construct, only be called by clone method
|
||||
//! Clone construct, used by clone method in subclasses
|
||||
HnswStreamerEntity(IndexStreamer::Stats &stats, const HNSWHeader &hd,
|
||||
size_t chunk_size, uint32_t node_index_mask_bits,
|
||||
uint32_t upper_neighbor_mask_bits, bool filter_same_key,
|
||||
@@ -480,14 +511,23 @@ class HnswStreamerEntity : public HnswEntity {
|
||||
return 0;
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Expose sync_chunks for subclass use
|
||||
inline void sync_node_chunks(size_t idx) const {
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, idx, &node_chunks_);
|
||||
}
|
||||
inline void sync_upper_neighbor_chunks(size_t idx) const {
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_UPPER_NEIGHBOR, idx,
|
||||
&upper_neighbor_chunks_);
|
||||
}
|
||||
|
||||
private:
|
||||
HnswStreamerEntity(const HnswStreamerEntity &) = delete;
|
||||
HnswStreamerEntity &operator=(const HnswStreamerEntity &) = delete;
|
||||
static constexpr uint64_t kUpperHashMemoryInflateRatio = 2.0f;
|
||||
|
||||
private:
|
||||
protected:
|
||||
IndexStreamer::Stats &stats_;
|
||||
HNSWHeader header_{};
|
||||
std::mutex mutex_{};
|
||||
size_t max_index_size_{0UL};
|
||||
uint32_t chunk_size_{kDefaultChunkSize};
|
||||
@@ -536,5 +576,434 @@ class HnswStreamerEntity : public HnswEntity {
|
||||
ChunkBroker::Pointer broker_{}; // chunk broker
|
||||
};
|
||||
|
||||
// --- Template specializations for typed MemoryBlock access ---
|
||||
|
||||
//! MmapMemoryBlock specialization: uses pointer-based Chunk::read
|
||||
template <>
|
||||
inline NeighborsT<MmapMemoryBlock>
|
||||
HnswStreamerEntity::get_neighbors_typed<MmapMemoryBlock>(level_t level,
|
||||
node_id_t id) const {
|
||||
Chunk *chunk = nullptr;
|
||||
size_t offset = 0UL;
|
||||
size_t nbr_size = neighbor_size_;
|
||||
if (level == 0UL) {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
offset =
|
||||
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
|
||||
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
|
||||
chunk = node_chunks_[chunk_idx].get();
|
||||
} else {
|
||||
auto p = get_upper_neighbor_chunk_loc(level, id);
|
||||
chunk = upper_neighbor_chunks_[p.first].get();
|
||||
offset = p.second;
|
||||
nbr_size = upper_neighbor_size_;
|
||||
}
|
||||
ailego_assert_with(offset < chunk->data_size(), "invalid chunk offset");
|
||||
const void *ptr = nullptr;
|
||||
size_t ret = chunk->read(offset, &ptr, nbr_size);
|
||||
if (ailego_unlikely(ret != nbr_size)) {
|
||||
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
|
||||
return NeighborsT<MmapMemoryBlock>();
|
||||
}
|
||||
MmapMemoryBlock block(const_cast<void *>(ptr));
|
||||
return NeighborsT<MmapMemoryBlock>(std::move(block));
|
||||
}
|
||||
|
||||
//! BufferPoolMemoryBlock specialization: uses MemoryBlock-based Chunk::read
|
||||
template <>
|
||||
inline NeighborsT<BufferPoolMemoryBlock>
|
||||
HnswStreamerEntity::get_neighbors_typed<BufferPoolMemoryBlock>(
|
||||
level_t level, node_id_t id) const {
|
||||
Chunk *chunk = nullptr;
|
||||
size_t offset = 0UL;
|
||||
size_t nbr_size = neighbor_size_;
|
||||
if (level == 0UL) {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
offset =
|
||||
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
|
||||
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
|
||||
chunk = node_chunks_[chunk_idx].get();
|
||||
} else {
|
||||
auto p = get_upper_neighbor_chunk_loc(level, id);
|
||||
chunk = upper_neighbor_chunks_[p.first].get();
|
||||
offset = p.second;
|
||||
nbr_size = upper_neighbor_size_;
|
||||
}
|
||||
ailego_assert_with(offset < chunk->data_size(), "invalid chunk offset");
|
||||
IndexStorage::MemoryBlock mem_block;
|
||||
size_t ret = chunk->read(offset, mem_block, nbr_size);
|
||||
if (ailego_unlikely(ret != nbr_size)) {
|
||||
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
|
||||
return NeighborsT<BufferPoolMemoryBlock>();
|
||||
}
|
||||
BufferPoolMemoryBlock block(mem_block.buffer_pool_handle_,
|
||||
mem_block.buffer_block_id_, mem_block.data_);
|
||||
mem_block.buffer_pool_handle_ = nullptr;
|
||||
return NeighborsT<BufferPoolMemoryBlock>(std::move(block));
|
||||
}
|
||||
|
||||
//! MmapMemoryBlock specialization for batch get_vector
|
||||
template <>
|
||||
inline int HnswStreamerEntity::get_vector_typed<MmapMemoryBlock>(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<MmapMemoryBlock> &vec_blocks) const {
|
||||
vec_blocks.resize(count);
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
auto loc = get_vector_chunk_loc(ids[i]);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
|
||||
"invalid chunk offset");
|
||||
size_t read_size = vector_size();
|
||||
const void *ptr = nullptr;
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, read_size);
|
||||
if (ailego_unlikely(ret != read_size)) {
|
||||
LOG_ERROR("Read vector failed, offset=%u, read size=%zu, ret=%zu",
|
||||
loc.second, read_size, ret);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
vec_blocks[i].reset(const_cast<void *>(ptr));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! BufferPoolMemoryBlock specialization for batch get_vector
|
||||
template <>
|
||||
inline int HnswStreamerEntity::get_vector_typed<BufferPoolMemoryBlock>(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<BufferPoolMemoryBlock> &vec_blocks) const {
|
||||
vec_blocks.resize(count);
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
auto loc = get_vector_chunk_loc(ids[i]);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
|
||||
"invalid chunk offset");
|
||||
size_t read_size = vector_size();
|
||||
IndexStorage::MemoryBlock mem_block;
|
||||
size_t ret =
|
||||
node_chunks_[loc.first]->read(loc.second, mem_block, read_size);
|
||||
if (ailego_unlikely(ret != read_size)) {
|
||||
LOG_ERROR("Read vector failed, offset=%u, read size=%zu, ret=%zu",
|
||||
loc.second, read_size, ret);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
vec_blocks[i] =
|
||||
BufferPoolMemoryBlock(mem_block.buffer_pool_handle_,
|
||||
mem_block.buffer_block_id_, mem_block.data_);
|
||||
mem_block.buffer_pool_handle_ = nullptr;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! MmapMemoryBlock specialization for get_key
|
||||
template <>
|
||||
inline key_t HnswStreamerEntity::get_key_typed<MmapMemoryBlock>(
|
||||
node_id_t id) const {
|
||||
if (!use_key_info_map_) {
|
||||
return id;
|
||||
}
|
||||
auto loc = get_key_chunk_loc(id);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
|
||||
"invalid chunk offset");
|
||||
const void *ptr = nullptr;
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, sizeof(key_t));
|
||||
if (ailego_unlikely(ret != sizeof(key_t))) {
|
||||
LOG_ERROR("Read key failed, ret=%zu", ret);
|
||||
return kInvalidKey;
|
||||
}
|
||||
return *reinterpret_cast<const key_t *>(ptr);
|
||||
}
|
||||
|
||||
//! BufferPoolMemoryBlock specialization for get_key
|
||||
template <>
|
||||
inline key_t HnswStreamerEntity::get_key_typed<BufferPoolMemoryBlock>(
|
||||
node_id_t id) const {
|
||||
if (!use_key_info_map_) {
|
||||
return id;
|
||||
}
|
||||
auto loc = get_key_chunk_loc(id);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(loc.second < node_chunks_[loc.first]->data_size(),
|
||||
"invalid chunk offset");
|
||||
IndexStorage::MemoryBlock key_block;
|
||||
size_t ret =
|
||||
node_chunks_[loc.first]->read(loc.second, key_block, sizeof(key_t));
|
||||
if (ailego_unlikely(ret != sizeof(key_t))) {
|
||||
LOG_ERROR("Read key failed, ret=%zu", ret);
|
||||
return kInvalidKey;
|
||||
}
|
||||
return *reinterpret_cast<const key_t *>(key_block.data());
|
||||
}
|
||||
|
||||
//! Typed entity subclass for mmap mode.
|
||||
//! Caches chunk base addresses to eliminate virtual function calls on the
|
||||
//! search hot path. For mmap mode, chunk data is memory-mapped at init time,
|
||||
//! so we can directly compute pointers via base_addr + offset.
|
||||
class HnswMmapStreamerEntity : public HnswStreamerEntity {
|
||||
public:
|
||||
using MemoryBlock = MmapMemoryBlock;
|
||||
using TypedNeighbors = NeighborsT<MmapMemoryBlock>;
|
||||
|
||||
using HnswStreamerEntity::HnswStreamerEntity;
|
||||
|
||||
HnswStorageMode storage_mode() const override {
|
||||
return HnswStorageMode::kMmap;
|
||||
}
|
||||
|
||||
//! Override clone to return correct subclass type, so that
|
||||
//! static_cast<const HnswMmapStreamerEntity&> in the algorithm is safe.
|
||||
const HnswEntity::Pointer clone() const override;
|
||||
|
||||
inline TypedNeighbors get_neighbors_typed(level_t level, node_id_t id) const {
|
||||
if (level == 0UL) {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset =
|
||||
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
|
||||
const char *base = get_node_chunk_base(chunk_idx);
|
||||
MmapMemoryBlock block(const_cast<char *>(base + offset));
|
||||
return TypedNeighbors(std::move(block));
|
||||
}
|
||||
// Upper level: use index to locate chunk and offset
|
||||
auto it = upper_neighbor_index_->find(id);
|
||||
ailego_assert_abort(it != upper_neighbor_index_->end(),
|
||||
"Get upper neighbor header failed");
|
||||
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
|
||||
uint32_t chunk_idx = (meta->index) >> upper_neighbor_mask_bits_;
|
||||
uint32_t offset = (((meta->index) & upper_neighbor_mask_) + level - 1) *
|
||||
upper_neighbor_size_;
|
||||
const char *base = get_upper_neighbor_chunk_base(chunk_idx);
|
||||
MmapMemoryBlock block(const_cast<char *>(base + offset));
|
||||
return TypedNeighbors(std::move(block));
|
||||
}
|
||||
|
||||
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
|
||||
std::vector<MmapMemoryBlock> &vec_blocks) const {
|
||||
vec_blocks.resize(count);
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
uint32_t chunk_idx = ids[i] >> node_index_mask_bits_;
|
||||
uint32_t offset = (ids[i] & node_index_mask_) * node_size();
|
||||
const char *base = get_node_chunk_base(chunk_idx);
|
||||
vec_blocks[i].reset(const_cast<char *>(base + offset));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline key_t get_key_typed(node_id_t id) const {
|
||||
if (!use_key_info_map_) {
|
||||
return id;
|
||||
}
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
|
||||
const char *base = get_node_chunk_base(chunk_idx);
|
||||
return *reinterpret_cast<const key_t *>(base + offset);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Get cached base address for a node chunk, syncing if needed
|
||||
inline const char *get_node_chunk_base(uint32_t chunk_idx) const {
|
||||
if (ailego_unlikely(chunk_idx >= node_chunk_bases_.size())) {
|
||||
sync_node_chunk_bases(chunk_idx);
|
||||
}
|
||||
return node_chunk_bases_[chunk_idx];
|
||||
}
|
||||
|
||||
//! Get cached base address for an upper neighbor chunk, syncing if needed
|
||||
inline const char *get_upper_neighbor_chunk_base(uint32_t chunk_idx) const {
|
||||
if (ailego_unlikely(chunk_idx >= upper_neighbor_chunk_bases_.size())) {
|
||||
sync_upper_neighbor_chunk_bases(chunk_idx);
|
||||
}
|
||||
return upper_neighbor_chunk_bases_[chunk_idx];
|
||||
}
|
||||
|
||||
//! Sync node chunk base addresses up to the given index
|
||||
void sync_node_chunk_bases(uint32_t chunk_idx) const {
|
||||
sync_node_chunks(chunk_idx);
|
||||
const auto &chunks = node_chunks_;
|
||||
for (size_t i = node_chunk_bases_.size(); i <= chunk_idx; ++i) {
|
||||
const void *ptr = nullptr;
|
||||
chunks[i]->read(0, &ptr, 1);
|
||||
node_chunk_bases_.push_back(static_cast<const char *>(ptr));
|
||||
}
|
||||
}
|
||||
|
||||
//! Sync upper neighbor chunk base addresses up to the given index
|
||||
void sync_upper_neighbor_chunk_bases(uint32_t chunk_idx) const {
|
||||
sync_upper_neighbor_chunks(chunk_idx);
|
||||
const auto &chunks = upper_neighbor_chunks_;
|
||||
for (size_t i = upper_neighbor_chunk_bases_.size(); i <= chunk_idx; ++i) {
|
||||
const void *ptr = nullptr;
|
||||
chunks[i]->read(0, &ptr, 1);
|
||||
upper_neighbor_chunk_bases_.push_back(static_cast<const char *>(ptr));
|
||||
}
|
||||
}
|
||||
|
||||
mutable std::vector<const char *> node_chunk_bases_{};
|
||||
mutable std::vector<const char *> upper_neighbor_chunk_bases_{};
|
||||
};
|
||||
|
||||
//! Typed entity subclass for buffer pool mode.
|
||||
class HnswBufferPoolStreamerEntity : public HnswStreamerEntity {
|
||||
public:
|
||||
using MemoryBlock = BufferPoolMemoryBlock;
|
||||
using TypedNeighbors = NeighborsT<BufferPoolMemoryBlock>;
|
||||
|
||||
using HnswStreamerEntity::HnswStreamerEntity;
|
||||
|
||||
HnswStorageMode storage_mode() const override {
|
||||
return HnswStorageMode::kBufferPool;
|
||||
}
|
||||
|
||||
inline TypedNeighbors get_neighbors_typed(level_t level, node_id_t id) const {
|
||||
return HnswStreamerEntity::get_neighbors_typed<BufferPoolMemoryBlock>(level,
|
||||
id);
|
||||
}
|
||||
|
||||
inline int get_vector_typed(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<BufferPoolMemoryBlock> &vec_blocks) const {
|
||||
return HnswStreamerEntity::get_vector_typed<BufferPoolMemoryBlock>(
|
||||
ids, count, vec_blocks);
|
||||
}
|
||||
|
||||
inline key_t get_key_typed(node_id_t id) const {
|
||||
return HnswStreamerEntity::get_key_typed<BufferPoolMemoryBlock>(id);
|
||||
}
|
||||
};
|
||||
|
||||
//! Typed entity subclass for contiguous memory mode.
|
||||
//! Allocates contiguous memory (with hugepage/THP support) and copies all
|
||||
//! chunk data into it. Access is via a single base pointer + offset,
|
||||
//! eliminating chunk-level indirection and maximizing memory locality.
|
||||
class HnswContiguousStreamerEntity : public HnswMmapStreamerEntity {
|
||||
public:
|
||||
using HnswMmapStreamerEntity::HnswMmapStreamerEntity;
|
||||
|
||||
HnswStorageMode storage_mode() const override {
|
||||
return HnswStorageMode::kContiguous;
|
||||
}
|
||||
|
||||
//! Override clone to return correct subclass type.
|
||||
//! Cloned entity shares contiguous memory via shared_ptr.
|
||||
const HnswEntity::Pointer clone() const override;
|
||||
|
||||
~HnswContiguousStreamerEntity() = default;
|
||||
|
||||
//! Build contiguous memory from chunks after open.
|
||||
//! Must be called after the entity is fully opened and all chunks are loaded.
|
||||
int build_contiguous_memory();
|
||||
|
||||
//! Degrade to mmap mode by releasing contiguous memory and falling back
|
||||
//! to chunk-based access.
|
||||
void degrade_to_mmap() {
|
||||
node_memory_.reset();
|
||||
node_base_ = nullptr;
|
||||
upper_neighbor_memory_.reset();
|
||||
upper_neighbor_base_ = nullptr;
|
||||
upper_chunk_offsets_.clear();
|
||||
LOG_INFO("HNSW contiguous entity degraded to mmap mode for insertion");
|
||||
}
|
||||
|
||||
bool is_contiguous() const {
|
||||
return node_base_ != nullptr;
|
||||
}
|
||||
|
||||
int add_vector(level_t level, key_t key, const void *vec,
|
||||
node_id_t *id) override {
|
||||
if (ailego_unlikely(is_contiguous())) degrade_to_mmap();
|
||||
return HnswMmapStreamerEntity::add_vector(level, key, vec, id);
|
||||
}
|
||||
|
||||
int add_vector_with_id(level_t level, node_id_t id,
|
||||
const void *vec) override {
|
||||
if (ailego_unlikely(is_contiguous())) degrade_to_mmap();
|
||||
return HnswMmapStreamerEntity::add_vector_with_id(level, id, vec);
|
||||
}
|
||||
|
||||
inline TypedNeighbors get_neighbors_typed(level_t level, node_id_t id) const {
|
||||
if (ailego_likely(node_base_ != nullptr)) {
|
||||
if (level == 0UL) {
|
||||
const char *ptr = node_base_ + static_cast<size_t>(id) * node_size() +
|
||||
vector_size() + sizeof(key_t);
|
||||
MmapMemoryBlock block(const_cast<char *>(ptr));
|
||||
return TypedNeighbors(std::move(block));
|
||||
}
|
||||
// Upper level: use index to locate global offset
|
||||
auto it = upper_neighbor_index_->find(id);
|
||||
ailego_assert_abort(it != upper_neighbor_index_->end(),
|
||||
"Get upper neighbor header failed");
|
||||
auto meta = reinterpret_cast<const UpperNeighborIndexMeta *>(&it->second);
|
||||
uint32_t chunk_idx = (meta->index) >> upper_neighbor_mask_bits_;
|
||||
uint32_t local_idx = (meta->index) & upper_neighbor_mask_;
|
||||
size_t global_offset =
|
||||
upper_chunk_offsets_[chunk_idx] +
|
||||
static_cast<size_t>(local_idx + level - 1) * upper_neighbor_size_;
|
||||
const char *ptr = upper_neighbor_base_ + global_offset;
|
||||
MmapMemoryBlock block(const_cast<char *>(ptr));
|
||||
return TypedNeighbors(std::move(block));
|
||||
}
|
||||
return HnswMmapStreamerEntity::get_neighbors_typed(level, id);
|
||||
}
|
||||
|
||||
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
|
||||
std::vector<MmapMemoryBlock> &vec_blocks) const {
|
||||
if (ailego_likely(node_base_ != nullptr)) {
|
||||
vec_blocks.resize(count);
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
const char *ptr =
|
||||
node_base_ + static_cast<size_t>(ids[i]) * node_size();
|
||||
vec_blocks[i].reset(const_cast<char *>(ptr));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
return HnswMmapStreamerEntity::get_vector_typed(ids, count, vec_blocks);
|
||||
}
|
||||
|
||||
inline key_t get_key_typed(node_id_t id) const {
|
||||
if (ailego_likely(node_base_ != nullptr)) {
|
||||
if (!use_key_info_map_) {
|
||||
return id;
|
||||
}
|
||||
const char *ptr =
|
||||
node_base_ + static_cast<size_t>(id) * node_size() + vector_size();
|
||||
return *reinterpret_cast<const key_t *>(ptr);
|
||||
}
|
||||
return HnswMmapStreamerEntity::get_key_typed(id);
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Custom deleter for contiguous memory (munmap / _aligned_free / free)
|
||||
//! Used by shared_ptr to properly release mmap'd memory.
|
||||
struct ContiguousDeleter {
|
||||
size_t size;
|
||||
void operator()(char *ptr) const {
|
||||
if (!ptr) return;
|
||||
#if defined(__linux__) || defined(__APPLE__)
|
||||
::munmap(ptr, size);
|
||||
#elif defined(_WIN32)
|
||||
::_aligned_free(ptr);
|
||||
#else
|
||||
std::free(ptr);
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
//! Shared ownership of contiguous memory (enables zero-copy clone)
|
||||
std::shared_ptr<char> node_memory_{};
|
||||
std::shared_ptr<char> upper_neighbor_memory_{};
|
||||
|
||||
//! Raw pointers for hot-path access (derived from shared_ptr)
|
||||
char *node_base_{nullptr};
|
||||
char *upper_neighbor_base_{nullptr};
|
||||
|
||||
//! Cumulative offsets for each upper neighbor chunk in contiguous memory
|
||||
std::vector<size_t> upper_chunk_offsets_{};
|
||||
|
||||
private:
|
||||
//! Allocate contiguous memory with hugepage/THP support
|
||||
static char *allocate_contiguous(size_t size);
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,11 @@
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
|
||||
cc_library(
|
||||
NAME core_knn_vamana
|
||||
STATIC SHARED STRICT ALWAYS_LINK
|
||||
SRCS *.cc
|
||||
LIBS core_framework core_knn_hnsw sparsehash
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
@@ -0,0 +1,465 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "vamana_algorithm.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
// ============================================================================
|
||||
// add_node: Insert a new node into the Vamana graph.
|
||||
//
|
||||
// Algorithm (from DiskANN / Vamana paper):
|
||||
// 1. GreedySearch from entry_point to find candidate neighbors
|
||||
// 2. RobustPrune to select diverse neighbors for the new node
|
||||
// 3. Update the new node's neighbor list
|
||||
// 4. For each new neighbor, add reverse link; if over-degree, RobustPrune
|
||||
// 5. If this is the first node, set it as entry point
|
||||
// ============================================================================
|
||||
template <typename EntityType>
|
||||
int VamanaAlgorithm<EntityType>::add_node(node_id_t id, VamanaContext *ctx) {
|
||||
// Lazily initialize distance storage on first insert
|
||||
entity_.ensure_dist_storage();
|
||||
|
||||
spin_lock_.lock();
|
||||
auto entry_point = entity_.entry_point();
|
||||
|
||||
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
|
||||
entity_.update_entry_point(id);
|
||||
spin_lock_.unlock();
|
||||
return 0;
|
||||
}
|
||||
spin_lock_.unlock();
|
||||
|
||||
// Step 1: GreedySearch to find candidate neighbors
|
||||
uint32_t search_list_size = entity_.search_list_size();
|
||||
ctx->topk_heap().clear();
|
||||
ctx->topk_heap().limit(search_list_size);
|
||||
ctx->dist_calculator().clear_compare_cnt();
|
||||
|
||||
// Set query to the new node's vector
|
||||
const void *query_vec = entity_.get_vector(id);
|
||||
if (ailego_unlikely(query_vec == nullptr)) {
|
||||
LOG_ERROR("Failed to get vector for node %u", id);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
ctx->reset_query(query_vec);
|
||||
|
||||
greedy_search(entry_point, ctx);
|
||||
|
||||
// Step 2: RobustPrune to select diverse neighbors
|
||||
auto &topk_heap = ctx->topk_heap();
|
||||
robust_prune(id, topk_heap, entity_.alpha(), entity_.max_degree(), ctx);
|
||||
// Copy result before reverse updates (which also call robust_prune)
|
||||
auto pruned_neighbors = ctx->prune_result();
|
||||
|
||||
// Step 3: Update the new node's neighbor list and distances
|
||||
entity_.update_neighbors(id, pruned_neighbors);
|
||||
entity_.update_neighbor_dists(id, pruned_neighbors);
|
||||
|
||||
// Step 4: Reverse-link updates
|
||||
update_neighbors_and_reverse_links(id, pruned_neighbors, ctx);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// search: Greedy search for approximate nearest neighbors.
|
||||
// ============================================================================
|
||||
template <typename EntityType>
|
||||
int VamanaAlgorithm<EntityType>::search(VamanaContext *ctx) const {
|
||||
spin_lock_.lock();
|
||||
auto entry_point = entity_.entry_point();
|
||||
spin_lock_.unlock();
|
||||
|
||||
if (ailego_unlikely(entry_point == kInvalidNodeId)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
auto &topk_heap = ctx->topk_heap();
|
||||
topk_heap.clear();
|
||||
|
||||
// Use ef (query-time parameter) instead of entity.search_list_size()
|
||||
// (build-time L parameter). search_list_size controls construction;
|
||||
// ef controls search quality and is user-configurable at query time.
|
||||
uint32_t ef_search = std::max(static_cast<uint32_t>(ctx->topk()), ctx->ef());
|
||||
topk_heap.limit(ef_search);
|
||||
|
||||
greedy_search(entry_point, ctx);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// greedy_search: Beam search from entry_point.
|
||||
//
|
||||
// Maintains a candidate min-heap (ordered by distance) and a visited set.
|
||||
// At each step, pops the closest unvisited candidate, expands its neighbors,
|
||||
// and adds unvisited neighbors to both the candidate heap and the topk heap.
|
||||
// Stops when the closest candidate is farther than the worst in topk, or
|
||||
// when the scan limit is reached.
|
||||
// ============================================================================
|
||||
template <typename EntityType>
|
||||
void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
|
||||
VamanaContext *ctx) const {
|
||||
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
|
||||
VamanaDistCalculator &dc = ctx->dist_calculator();
|
||||
VisitFilter &visit = ctx->visit_filter();
|
||||
CandidateHeap &candidates = ctx->candidates();
|
||||
auto &topk_heap = ctx->topk_heap();
|
||||
|
||||
const IndexFilter &index_filter =
|
||||
static_cast<const IndexContext *>(ctx)->filter();
|
||||
std::function<bool(node_id_t)> filter = [](node_id_t) { return false; };
|
||||
if (index_filter.is_valid()) {
|
||||
filter = [&](node_id_t id) {
|
||||
return index_filter(entity.get_key_typed(id));
|
||||
};
|
||||
}
|
||||
|
||||
candidates.clear();
|
||||
visit.clear();
|
||||
|
||||
// Initialize with entry point using batch_dist (single-element batch).
|
||||
// We must NOT use dc.dist(entry_point) here because dist() calls
|
||||
// distance_() which is squared_euclidean_int8_distance (sign/abs trick,
|
||||
// expects two raw int8 inputs), but query_ has been preprocessed by
|
||||
// reset_query (+128 shift to uint8). batch_dist() correctly calls
|
||||
// batch_distance_() which expects the preprocessed uint8 query.
|
||||
dist_t entry_dist = dc.batch_dist(entry_point);
|
||||
if (ailego_unlikely(dc.error())) {
|
||||
return;
|
||||
}
|
||||
visit.set_visited(entry_point);
|
||||
if (!filter(entry_point)) {
|
||||
topk_heap.emplace(entry_point, entry_dist);
|
||||
}
|
||||
candidates.emplace(entry_point, entry_dist);
|
||||
|
||||
// Pre-allocate temporary vectors outside the hot loop to avoid
|
||||
// per-iteration heap allocations. Sized to max_degree initially;
|
||||
// resized inside the loop if actual neighbor count exceeds this.
|
||||
uint32_t buf_capacity = entity.max_degree();
|
||||
std::vector<node_id_t> neighbor_ids(buf_capacity);
|
||||
std::vector<MemBlockType> neighbor_vec_blocks;
|
||||
neighbor_vec_blocks.reserve(buf_capacity);
|
||||
std::vector<float> dists(buf_capacity);
|
||||
std::vector<const void *> neighbor_vecs(buf_capacity);
|
||||
|
||||
while (!candidates.empty() && !ctx->reach_scan_limit()) {
|
||||
auto top = candidates.begin();
|
||||
node_id_t current_node = top->first;
|
||||
dist_t current_dist = top->second;
|
||||
|
||||
// Early termination: if the closest candidate is worse than the worst
|
||||
// result in a full topk heap, we won't find anything better.
|
||||
if (topk_heap.full() && current_dist > topk_heap[0].second) {
|
||||
break;
|
||||
}
|
||||
|
||||
candidates.pop();
|
||||
|
||||
// Expand neighbors using typed access (no virtual dispatch)
|
||||
const auto neighbors = entity.get_neighbors_typed(current_node);
|
||||
ailego_prefetch(neighbors.data);
|
||||
|
||||
// Resize buffers if this node has more neighbors than expected
|
||||
if (neighbors.size() > buf_capacity) {
|
||||
buf_capacity = neighbors.size();
|
||||
neighbor_ids.resize(buf_capacity);
|
||||
dists.resize(buf_capacity);
|
||||
neighbor_vecs.resize(buf_capacity);
|
||||
}
|
||||
|
||||
// Collect unvisited neighbors (reuse pre-allocated buffer)
|
||||
uint32_t unvisited_count = 0;
|
||||
for (uint32_t i = 0; i < neighbors.size(); ++i) {
|
||||
node_id_t node = neighbors[i];
|
||||
if (visit.visited(node)) continue;
|
||||
visit.set_visited(node);
|
||||
neighbor_ids[unvisited_count++] = node;
|
||||
}
|
||||
if (unvisited_count == 0) continue;
|
||||
|
||||
// Batch fetch vectors using typed access (reuse pre-allocated buffer)
|
||||
neighbor_vec_blocks.clear();
|
||||
int ret = entity.get_vector_typed(neighbor_ids.data(), unvisited_count,
|
||||
neighbor_vec_blocks);
|
||||
if (ailego_unlikely(ret != 0)) break;
|
||||
|
||||
// Prefetch for better cache performance
|
||||
static constexpr uint32_t PREFETCH_BATCH = 2;
|
||||
static constexpr uint32_t PREFETCH_STEP = 2;
|
||||
for (uint32_t i = 0;
|
||||
i < std::min(PREFETCH_BATCH * PREFETCH_STEP, unvisited_count); ++i) {
|
||||
ailego_prefetch(neighbor_vec_blocks[i].data());
|
||||
}
|
||||
|
||||
// Batch distance computation (reuse pre-allocated buffers)
|
||||
for (uint32_t i = 0; i < unvisited_count; ++i) {
|
||||
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
|
||||
}
|
||||
dc.batch_dist(neighbor_vecs.data(), unvisited_count, dists.data());
|
||||
|
||||
// Update candidates and topk.
|
||||
// Unlike vanilla DiskANN which inserts all unvisited neighbors into
|
||||
// the candidate queue unconditionally, we apply an early-pruning
|
||||
// optimization: a neighbor is only inserted into the candidate queue
|
||||
// (and topk_heap) if it could potentially improve the final results,
|
||||
// i.e. either the topk heap is not yet full, or the neighbor is closer
|
||||
// than the current worst result. This avoids expanding clearly
|
||||
// unpromising branches and reduces the candidate queue size.
|
||||
for (uint32_t i = 0; i < unvisited_count; ++i) {
|
||||
node_id_t node = neighbor_ids[i];
|
||||
dist_t node_dist = dists[i];
|
||||
if ((!topk_heap.full()) || node_dist < topk_heap[0].second) {
|
||||
candidates.emplace(node, node_dist);
|
||||
if (!filter(node)) {
|
||||
topk_heap.emplace(node, node_dist);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// robust_prune: Select up to max_degree diverse neighbors from candidates.
|
||||
//
|
||||
// Faithfully follows DiskANN's occlude_list algorithm:
|
||||
// 1. Sort candidates by distance (ascending)
|
||||
// 2. Truncate to max_occlusion_size (DiskANN's maxc parameter)
|
||||
// 3. Multi-round alpha sweep: cur_alpha starts at 1.0, increments by *1.2
|
||||
// each round until reaching alpha. This progressively relaxes the
|
||||
// occlusion criterion.
|
||||
// 4. For each candidate, compute occlude_factor as:
|
||||
// max over all selected neighbors p of: dist(query, candidate) / dist(p,
|
||||
// candidate)
|
||||
// If occlude_factor > cur_alpha, the candidate is occluded in this round.
|
||||
// 5. After all rounds, if _saturate_graph and alpha > 1, fill remaining
|
||||
// slots with any un-selected candidates.
|
||||
// ============================================================================
|
||||
template <typename EntityType>
|
||||
void VamanaAlgorithm<EntityType>::robust_prune(node_id_t id,
|
||||
TopkHeap &candidates,
|
||||
float alpha, uint32_t max_degree,
|
||||
VamanaContext *ctx) const {
|
||||
auto &result = ctx->prune_result();
|
||||
result.clear();
|
||||
|
||||
if (candidates.size() == 0) return;
|
||||
|
||||
// Sort candidates by distance (ascending — closest first)
|
||||
candidates.sort();
|
||||
|
||||
VamanaDistCalculator &dc = ctx->dist_calculator();
|
||||
size_t n = candidates.size();
|
||||
|
||||
// Truncate to max_occlusion_size (DiskANN's maxc parameter)
|
||||
size_t maxc = entity_.max_occlusion_size();
|
||||
if (maxc > 0 && n > maxc) {
|
||||
n = maxc;
|
||||
}
|
||||
|
||||
// Pre-cache all candidate vectors at once
|
||||
auto &vec_cache = ctx->prune_vec_cache();
|
||||
vec_cache.resize(n);
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
vec_cache[i] = entity_.get_vector(candidates[i].first);
|
||||
}
|
||||
|
||||
// occlude_factor: tracks the maximum occlusion ratio for each candidate
|
||||
// (DiskANN: occlude_factor[t] = max over selected p of
|
||||
// dist_to_query/dist_to_p)
|
||||
auto &occlude_factor = ctx->prune_occlude_factor();
|
||||
occlude_factor.assign(n, 0.0f);
|
||||
|
||||
// Pre-allocated buffers for batch distance computation
|
||||
auto &batch_vecs = ctx->batch_vecs_buf();
|
||||
auto &batch_dists = ctx->batch_dists_buf();
|
||||
auto &batch_indices = ctx->batch_indices_buf();
|
||||
batch_vecs.resize(n);
|
||||
batch_dists.resize(n);
|
||||
batch_indices.resize(n);
|
||||
|
||||
// Multi-round alpha sweep (DiskANN: cur_alpha starts at 1, increments *1.2)
|
||||
float cur_alpha = 1.0f;
|
||||
while (cur_alpha <= alpha + 1e-6f && result.size() < max_degree) {
|
||||
for (size_t i = 0; i < n && result.size() < max_degree; ++i) {
|
||||
if (occlude_factor[i] > cur_alpha) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Mark as consumed so it won't be reconsidered
|
||||
occlude_factor[i] = std::numeric_limits<float>::max();
|
||||
|
||||
// Skip self-loops
|
||||
if (candidates[i].first == id) continue;
|
||||
|
||||
const void *selected_vec = vec_cache[i];
|
||||
if (ailego_unlikely(selected_vec == nullptr)) continue;
|
||||
|
||||
// Add this candidate as a neighbor
|
||||
node_id_t candidate_id = candidates[i].first;
|
||||
dist_t candidate_dist = candidates[i].second;
|
||||
result.emplace_back(candidate_id, candidate_dist);
|
||||
|
||||
// Update occlude_factor for remaining candidates
|
||||
// Collect candidates that haven't been consumed yet
|
||||
uint32_t batch_count = 0;
|
||||
for (size_t j = i + 1; j < n; ++j) {
|
||||
if (occlude_factor[j] > alpha) continue; // already fully occluded
|
||||
if (ailego_unlikely(vec_cache[j] == nullptr)) continue;
|
||||
batch_vecs[batch_count] = vec_cache[j];
|
||||
batch_indices[batch_count] = static_cast<uint32_t>(j);
|
||||
batch_count++;
|
||||
}
|
||||
|
||||
if (batch_count > 0) {
|
||||
// Batch compute distances from selected candidate to remaining
|
||||
dc.batch_dist_pair(selected_vec, batch_vecs.data(), batch_count,
|
||||
batch_dists.data());
|
||||
|
||||
// DiskANN (L2/Cosine):
|
||||
// occlude_factor[t] = max(occlude_factor[t], dist_to_query /
|
||||
// dist_between)
|
||||
// where dist_to_query = candidates[j].second (distance from query to j)
|
||||
// dist_between = batch_dists[k] (distance from selected to j)
|
||||
for (uint32_t k = 0; k < batch_count; ++k) {
|
||||
uint32_t j = batch_indices[k];
|
||||
float dist_selected_to_candidate = batch_dists[k];
|
||||
if (dist_selected_to_candidate == 0.0f) {
|
||||
occlude_factor[j] = std::numeric_limits<float>::max();
|
||||
} else {
|
||||
occlude_factor[j] =
|
||||
std::max(occlude_factor[j],
|
||||
candidates[j].second / dist_selected_to_candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cur_alpha *= 1.2f;
|
||||
}
|
||||
|
||||
// Saturate graph: if enabled and alpha > 1, fill remaining slots with any
|
||||
// un-selected candidates. This improves graph connectivity (better recall)
|
||||
// at the cost of slightly more distance computations during search.
|
||||
// Configurable via proxima.vamana.streamer.saturate_graph (default: false,
|
||||
// matching DiskANN's default behavior).
|
||||
if (entity_.saturate_graph() && alpha > 1.0f) {
|
||||
for (size_t i = 0; i < n && result.size() < max_degree; ++i) {
|
||||
if (candidates[i].first == id) continue;
|
||||
bool already_selected = false;
|
||||
for (const auto &r : result) {
|
||||
if (r.first == candidates[i].first) {
|
||||
already_selected = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!already_selected) {
|
||||
result.emplace_back(candidates[i].first, candidates[i].second);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// update_neighbors_and_reverse_links: For each new neighbor of `id`,
|
||||
// add a reverse link from neighbor back to `id`. If the neighbor's degree
|
||||
// exceeds max_degree, prune it using RobustPrune.
|
||||
// ============================================================================
|
||||
template <typename EntityType>
|
||||
void VamanaAlgorithm<EntityType>::update_neighbors_and_reverse_links(
|
||||
node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &new_neighbors,
|
||||
VamanaContext *ctx) {
|
||||
for (const auto &[neighbor_id, dist] : new_neighbors) {
|
||||
reverse_update_neighbor(id, neighbor_id, dist, ctx);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// reverse_update_neighbor: Add `id` as a neighbor of `neighbor_id`.
|
||||
// If neighbor_id already has max_degree neighbors, collect all neighbors
|
||||
// + the new one into a candidate set and RobustPrune.
|
||||
// ============================================================================
|
||||
template <typename EntityType>
|
||||
void VamanaAlgorithm<EntityType>::reverse_update_neighbor(node_id_t id,
|
||||
node_id_t neighbor_id,
|
||||
dist_t dist,
|
||||
VamanaContext *ctx) {
|
||||
std::lock_guard<std::mutex> lock(lock_pool_[neighbor_id & kLockMask]);
|
||||
|
||||
const Neighbors current_neighbors = entity_.get_neighbors(neighbor_id);
|
||||
uint32_t current_size = current_neighbors.size();
|
||||
uint32_t max_deg = entity_.max_degree();
|
||||
|
||||
// Check if `id` is already a neighbor
|
||||
for (uint32_t i = 0; i < current_size; ++i) {
|
||||
if (current_neighbors[i] == id) return;
|
||||
}
|
||||
|
||||
if (current_size < max_deg) {
|
||||
// Simply append and record distance
|
||||
entity_.add_neighbor(neighbor_id, current_size, id);
|
||||
entity_.set_neighbor_dist(neighbor_id, current_size, dist);
|
||||
return;
|
||||
}
|
||||
|
||||
// Need to prune: collect current neighbors + new node into candidates
|
||||
VamanaDistCalculator &dc = ctx->dist_calculator();
|
||||
|
||||
// Reuse update_heap from context instead of creating a new TopkHeap each time
|
||||
TopkHeap &prune_candidates = ctx->update_heap();
|
||||
prune_candidates.clear();
|
||||
prune_candidates.limit(max_deg + 1);
|
||||
|
||||
// Add existing neighbors — use cached distances when available
|
||||
const dist_t *cached_dists = entity_.get_neighbor_dists(neighbor_id);
|
||||
if (cached_dists != nullptr) {
|
||||
// Fast path: read distances from storage, no recomputation needed
|
||||
for (uint32_t i = 0; i < current_size; ++i) {
|
||||
prune_candidates.emplace(current_neighbors[i], cached_dists[i]);
|
||||
}
|
||||
} else {
|
||||
// Fallback: compute distances (first time or dist storage not loaded)
|
||||
const void *neighbor_vec = entity_.get_vector(neighbor_id);
|
||||
if (ailego_unlikely(neighbor_vec == nullptr)) return;
|
||||
for (uint32_t i = 0; i < current_size; ++i) {
|
||||
node_id_t nbr = current_neighbors[i];
|
||||
const void *nbr_vec = entity_.get_vector(nbr);
|
||||
if (ailego_unlikely(nbr_vec == nullptr)) continue;
|
||||
dist_t nbr_dist = dc.dist(neighbor_vec, nbr_vec);
|
||||
prune_candidates.emplace(nbr, nbr_dist);
|
||||
}
|
||||
}
|
||||
|
||||
// Add the new reverse link
|
||||
prune_candidates.emplace(id, dist);
|
||||
|
||||
// RobustPrune from neighbor_id's perspective
|
||||
robust_prune(neighbor_id, prune_candidates, entity_.alpha(), max_deg, ctx);
|
||||
|
||||
// Update neighbor_id's neighbor list and distances
|
||||
const auto &prune_result = ctx->prune_result();
|
||||
entity_.update_neighbors(neighbor_id, prune_result);
|
||||
entity_.update_neighbor_dists(neighbor_id, prune_result);
|
||||
}
|
||||
|
||||
// Explicit template instantiation for all entity types
|
||||
template class VamanaAlgorithm<VamanaMmapStreamerEntity>;
|
||||
template class VamanaAlgorithm<VamanaBufferPoolStreamerEntity>;
|
||||
template class VamanaAlgorithm<VamanaContiguousStreamerEntity>;
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,115 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include "vamana_context.h"
|
||||
#include "vamana_dist_calculator.h"
|
||||
#include "vamana_streamer_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
// Non-template base class providing a type-erased interface so that
|
||||
// VamanaStreamer can hold a pointer without knowing the EntityType.
|
||||
class VamanaAlgorithmBase {
|
||||
public:
|
||||
typedef std::unique_ptr<VamanaAlgorithmBase> UPointer;
|
||||
|
||||
virtual ~VamanaAlgorithmBase() = default;
|
||||
|
||||
virtual int cleanup() = 0;
|
||||
|
||||
// Insert a new node into the Vamana graph.
|
||||
// The node's vector must already be stored in the entity.
|
||||
virtual int add_node(node_id_t id, VamanaContext *ctx) = 0;
|
||||
|
||||
// Greedy search: find approximate nearest neighbors.
|
||||
virtual int search(VamanaContext *ctx) const = 0;
|
||||
|
||||
virtual int init() = 0;
|
||||
};
|
||||
|
||||
// Vamana graph algorithm, templated on EntityType for hot-path optimization.
|
||||
// EntityType should be VamanaMmapStreamerEntity,
|
||||
// VamanaBufferPoolStreamerEntity, or VamanaContiguousStreamerEntity.
|
||||
//
|
||||
// Core operations:
|
||||
// - GreedySearch: beam search from entry point, expanding best candidates
|
||||
// - RobustPrune: select diverse neighbors using alpha-based pruning
|
||||
// - add_node: insert + prune + reverse-link update
|
||||
template <typename EntityType>
|
||||
class VamanaAlgorithm : public VamanaAlgorithmBase {
|
||||
public:
|
||||
using MemBlockType = typename EntityType::MemoryBlock;
|
||||
|
||||
explicit VamanaAlgorithm(EntityType &entity)
|
||||
: entity_(entity), lock_pool_(kLockCnt) {}
|
||||
|
||||
~VamanaAlgorithm() override = default;
|
||||
|
||||
int cleanup() override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int init() override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Insert node `id` into the graph. Its vector must already be in the entity.
|
||||
int add_node(node_id_t id, VamanaContext *ctx) override;
|
||||
|
||||
// Greedy search from entry point. Results are stored in ctx->topk_heap().
|
||||
int search(VamanaContext *ctx) const override;
|
||||
|
||||
private:
|
||||
// GreedySearch: starting from entry_point, greedily expand the closest
|
||||
// unvisited candidate until the search list is exhausted or scan limit
|
||||
// is reached. Results accumulate in topk_heap.
|
||||
void greedy_search(node_id_t entry_point, VamanaContext *ctx) const;
|
||||
|
||||
// RobustPrune: given a candidate set (topk_heap), select up to max_degree
|
||||
// diverse neighbors using alpha-based distance comparison.
|
||||
// Result is stored in ctx->prune_result().
|
||||
void robust_prune(node_id_t id, TopkHeap &candidates, float alpha,
|
||||
uint32_t max_degree, VamanaContext *ctx) const;
|
||||
|
||||
// Update node's neighbors and handle reverse links.
|
||||
void update_neighbors_and_reverse_links(
|
||||
node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &new_neighbors,
|
||||
VamanaContext *ctx);
|
||||
|
||||
// Check if adding `id` as a reverse neighbor of `neighbor_id` requires
|
||||
// pruning, and if so, prune neighbor_id's neighbor list.
|
||||
void reverse_update_neighbor(node_id_t id, node_id_t neighbor_id, dist_t dist,
|
||||
VamanaContext *ctx);
|
||||
|
||||
private:
|
||||
VamanaAlgorithm(const VamanaAlgorithm &) = delete;
|
||||
VamanaAlgorithm &operator=(const VamanaAlgorithm &) = delete;
|
||||
|
||||
static constexpr uint32_t kLockCnt{1U << 8};
|
||||
static constexpr uint32_t kLockMask{kLockCnt - 1U};
|
||||
|
||||
EntityType &entity_;
|
||||
mutable ailego::SpinMutex spin_lock_{};
|
||||
std::vector<std::mutex> lock_pool_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,159 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "vamana_context.h"
|
||||
#include <random>
|
||||
#include "vamana_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
VamanaContext::VamanaContext(size_t dimension,
|
||||
const IndexMetric::Pointer &metric,
|
||||
const VamanaEntity::Pointer &entity)
|
||||
: IndexContext(metric),
|
||||
entity_(entity),
|
||||
dc_(entity.get(), metric, dimension),
|
||||
metric_(metric) {}
|
||||
|
||||
VamanaContext::VamanaContext(const IndexMetric::Pointer &metric,
|
||||
const VamanaEntity::Pointer &entity)
|
||||
: IndexContext(metric),
|
||||
entity_(entity),
|
||||
dc_(entity.get(), metric),
|
||||
metric_(metric) {}
|
||||
|
||||
VamanaContext::~VamanaContext() {
|
||||
visit_filter_.destroy();
|
||||
}
|
||||
|
||||
int VamanaContext::init(ContextType type) {
|
||||
int ret;
|
||||
uint32_t doc_cnt;
|
||||
|
||||
type_ = type;
|
||||
results_.resize(1);
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
update_heap_.limit(entity_->max_degree());
|
||||
|
||||
switch (type) {
|
||||
case kBuilderContext:
|
||||
ret = visit_filter_.init(VisitFilter::ByteMap, entity_->doc_cnt(),
|
||||
max_scan_num_, filter_negative_prob_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create visit filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
candidates_.limit(max_scan_num_);
|
||||
break;
|
||||
|
||||
case kSearcherContext:
|
||||
ret = visit_filter_.init(filter_mode_, entity_->doc_cnt(), max_scan_num_,
|
||||
filter_negative_prob_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create visit filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
candidates_.limit(max_scan_num_);
|
||||
break;
|
||||
|
||||
case kStreamerContext:
|
||||
doc_cnt = entity_->doc_cnt();
|
||||
max_scan_num_ = compute_max_scan_num(doc_cnt);
|
||||
reserve_max_doc_cnt_ = doc_cnt + compute_reserve_cnt(doc_cnt);
|
||||
ret = visit_filter_.init(filter_mode_, reserve_max_doc_cnt_,
|
||||
max_scan_num_, filter_negative_prob_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Create visit filter failed, mode %d", filter_mode_);
|
||||
return ret;
|
||||
}
|
||||
candidates_.limit(max_scan_num_);
|
||||
check_need_adjuct_ctx();
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaContext::update_context(ContextType type, const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &metric,
|
||||
const VamanaEntity::Pointer &entity,
|
||||
uint32_t magic_num) {
|
||||
if (magic_ == magic_num) {
|
||||
return 0;
|
||||
}
|
||||
type_ = type;
|
||||
entity_ = entity;
|
||||
metric_ = metric;
|
||||
magic_ = magic_num;
|
||||
dc_.update(entity.get(), metric, meta.dimension());
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaContext::update(const ailego::Params ¶ms) {
|
||||
uint32_t ef = ef_;
|
||||
params.get(PARAM_VAMANA_STREAMER_EF, &ef);
|
||||
ef_ = ef;
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
return 0;
|
||||
}
|
||||
|
||||
void VamanaContext::topk_to_result(uint32_t idx) {
|
||||
if (force_padding_topk_ && !topk_heap_.full() &&
|
||||
topk_heap_.size() < entity_->doc_cnt()) {
|
||||
this->fill_random_to_topk_full();
|
||||
}
|
||||
if (ailego_unlikely(topk_heap_.size() == 0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
int size = std::min(topk_, static_cast<uint32_t>(topk_heap_.size()));
|
||||
topk_heap_.sort();
|
||||
results_[idx].clear();
|
||||
|
||||
for (int i = 0; i < size; ++i) {
|
||||
auto score = topk_heap_[i].second;
|
||||
if (score > this->threshold()) {
|
||||
break;
|
||||
}
|
||||
node_id_t id = topk_heap_[i].first;
|
||||
if (fetch_vector_) {
|
||||
results_[idx].emplace_back(entity_->get_key(id), score, id,
|
||||
entity_->get_vector(id));
|
||||
} else {
|
||||
results_[idx].emplace_back(entity_->get_key(id), score, id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void VamanaContext::fill_random_to_topk_full() {
|
||||
std::mt19937 rng(42);
|
||||
uint32_t doc_cnt = entity_->doc_cnt();
|
||||
uint32_t max_attempts = doc_cnt * 2;
|
||||
uint32_t attempts = 0;
|
||||
while (!topk_heap_.full() && doc_cnt > 0 && attempts < max_attempts) {
|
||||
node_id_t random_id = rng() % doc_cnt;
|
||||
if (entity_->get_key(random_id) != kInvalidKey) {
|
||||
dist_t random_dist = dc_.dist(random_id);
|
||||
topk_heap_.emplace_back(random_id, random_dist);
|
||||
}
|
||||
++attempts;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,315 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_context.h>
|
||||
#include "utility/visit_filter.h"
|
||||
#include "vamana_dist_calculator.h"
|
||||
#include "vamana_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class VamanaContext : public IndexContext {
|
||||
public:
|
||||
typedef std::unique_ptr<VamanaContext> Pointer;
|
||||
|
||||
enum ContextType {
|
||||
kUnknownContext = 0,
|
||||
kSearcherContext = 1,
|
||||
kBuilderContext = 2,
|
||||
kStreamerContext = 3
|
||||
};
|
||||
|
||||
VamanaContext(size_t dimension, const IndexMetric::Pointer &metric,
|
||||
const VamanaEntity::Pointer &entity);
|
||||
|
||||
VamanaContext(const IndexMetric::Pointer &metric,
|
||||
const VamanaEntity::Pointer &entity);
|
||||
|
||||
virtual ~VamanaContext();
|
||||
|
||||
virtual void set_topk(uint32_t val) override {
|
||||
topk_ = val;
|
||||
topk_heap_.limit(std::max(val, ef_));
|
||||
}
|
||||
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
virtual uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
virtual void set_debug_mode(bool enable) override {
|
||||
debug_mode_ = enable;
|
||||
}
|
||||
virtual bool debug_mode(void) const override {
|
||||
return debug_mode_;
|
||||
}
|
||||
|
||||
virtual std::string debug_string(void) const override {
|
||||
char buf[4096];
|
||||
size_t size = snprintf(buf, sizeof(buf), "scan_cnt=%zu", get_scan_num());
|
||||
return std::string(buf, size);
|
||||
}
|
||||
|
||||
virtual int update(const ailego::Params ¶ms) override;
|
||||
|
||||
int init(ContextType type);
|
||||
|
||||
int update_context(ContextType type, const IndexMeta &meta,
|
||||
const IndexMetric::Pointer &metric,
|
||||
const VamanaEntity::Pointer &entity, uint32_t magic_num);
|
||||
|
||||
inline const VamanaEntity &get_entity() const {
|
||||
return *entity_;
|
||||
}
|
||||
|
||||
inline void resize_results(size_t size) {
|
||||
results_.resize(size);
|
||||
}
|
||||
|
||||
inline void topk_to_result() {
|
||||
topk_to_result(0);
|
||||
}
|
||||
|
||||
void topk_to_result(uint32_t idx);
|
||||
|
||||
inline void reset_query(const void *query) {
|
||||
if (auto query_preprocess_func = index_metric_->get_query_preprocess_func();
|
||||
query_preprocess_func != nullptr) {
|
||||
size_t dim = dc_.dimension();
|
||||
preprocess_buffer_.resize(dim);
|
||||
memcpy(preprocess_buffer_.data(), query, dim);
|
||||
query_preprocess_func(preprocess_buffer_.data(), dim);
|
||||
query = preprocess_buffer_.data();
|
||||
}
|
||||
dc_.reset_query(query);
|
||||
dc_.clear_compare_cnt();
|
||||
}
|
||||
|
||||
inline VamanaDistCalculator &dist_calculator() {
|
||||
return dc_;
|
||||
}
|
||||
inline TopkHeap &topk_heap() {
|
||||
return topk_heap_;
|
||||
}
|
||||
inline TopkHeap &update_heap() {
|
||||
return update_heap_;
|
||||
}
|
||||
inline VisitFilter &visit_filter() {
|
||||
return visit_filter_;
|
||||
}
|
||||
inline CandidateHeap &candidates() {
|
||||
return candidates_;
|
||||
}
|
||||
|
||||
// Pre-allocated buffers for robust_prune optimization
|
||||
inline std::vector<const void *> &prune_vec_cache() {
|
||||
return prune_vec_cache_;
|
||||
}
|
||||
inline std::vector<uint8_t> &prune_active() {
|
||||
return prune_active_;
|
||||
}
|
||||
inline std::vector<float> &prune_occlude_factor() {
|
||||
return prune_occlude_factor_;
|
||||
}
|
||||
inline std::vector<std::pair<node_id_t, dist_t>> &prune_result() {
|
||||
return prune_result_;
|
||||
}
|
||||
inline std::vector<const void *> &batch_vecs_buf() {
|
||||
return batch_vecs_buf_;
|
||||
}
|
||||
inline std::vector<float> &batch_dists_buf() {
|
||||
return batch_dists_buf_;
|
||||
}
|
||||
inline std::vector<uint32_t> &batch_indices_buf() {
|
||||
return batch_indices_buf_;
|
||||
}
|
||||
|
||||
inline void set_max_scan_num(uint32_t max_scan_num) {
|
||||
max_scan_num_ = max_scan_num;
|
||||
}
|
||||
inline void set_ef(uint32_t v) {
|
||||
ef_ = v;
|
||||
}
|
||||
|
||||
inline uint32_t ef() const {
|
||||
return ef_;
|
||||
}
|
||||
inline void set_max_scan_ratio(float v) {
|
||||
max_scan_ratio_ = v;
|
||||
}
|
||||
virtual void set_magic(uint32_t v) {
|
||||
magic_ = v;
|
||||
}
|
||||
virtual void set_force_padding_topk(bool v) {
|
||||
force_padding_topk_ = v;
|
||||
}
|
||||
|
||||
void set_bruteforce_threshold(uint32_t v) override {
|
||||
bruteforce_threshold_ = v;
|
||||
}
|
||||
inline uint32_t get_bruteforce_threshold() const {
|
||||
return bruteforce_threshold_;
|
||||
}
|
||||
|
||||
void set_fetch_vector(bool v) override {
|
||||
fetch_vector_ = v;
|
||||
}
|
||||
bool fetch_vector() const override {
|
||||
return fetch_vector_;
|
||||
}
|
||||
|
||||
void set_max_scan_limit(size_t v) {
|
||||
max_scan_limit_ = v;
|
||||
}
|
||||
void set_min_scan_limit(size_t v) {
|
||||
min_scan_limit_ = v;
|
||||
}
|
||||
|
||||
void set_filter_mode(VisitFilter::Mode mode) {
|
||||
filter_mode_ = mode;
|
||||
}
|
||||
void set_filter_negative_probability(float prob) {
|
||||
filter_negative_prob_ = prob;
|
||||
}
|
||||
|
||||
void reset(void) override {
|
||||
dc_.clear();
|
||||
for (auto &it : results_) {
|
||||
it.clear();
|
||||
}
|
||||
IndexContext::reset_filter();
|
||||
IndexContext::reset_threshold();
|
||||
IndexContext::set_fetch_vector(false);
|
||||
}
|
||||
|
||||
inline void check_need_adjuct_ctx(void) {
|
||||
check_need_adjuct_ctx(entity_->doc_cnt());
|
||||
}
|
||||
|
||||
inline void check_need_adjuct_ctx(uint32_t doc_cnt) {
|
||||
if (ailego_unlikely(doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_)) {
|
||||
while (doc_cnt + kTriggerReserveCnt > reserve_max_doc_cnt_) {
|
||||
reserve_max_doc_cnt_ =
|
||||
reserve_max_doc_cnt_ + compute_reserve_cnt(reserve_max_doc_cnt_);
|
||||
}
|
||||
uint32_t max_scan_cnt = compute_max_scan_num(reserve_max_doc_cnt_);
|
||||
max_scan_num_ = max_scan_cnt;
|
||||
visit_filter_.reset(reserve_max_doc_cnt_, max_scan_cnt);
|
||||
candidates_.clear();
|
||||
candidates_.limit(max_scan_num_);
|
||||
}
|
||||
}
|
||||
|
||||
inline size_t get_scan_num() const {
|
||||
return dc_.compare_cnt();
|
||||
}
|
||||
|
||||
inline uint64_t reach_scan_limit() const {
|
||||
return dc_.compare_cnt() >= max_scan_num_;
|
||||
}
|
||||
|
||||
inline bool error() const {
|
||||
return dc_.error();
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
dc_.clear();
|
||||
for (auto &it : results_) {
|
||||
it.clear();
|
||||
}
|
||||
}
|
||||
|
||||
inline uint32_t topk() const override {
|
||||
return topk_;
|
||||
}
|
||||
|
||||
inline void update_dist_caculator_distance(
|
||||
const IndexMetric::MatrixDistance &distance,
|
||||
const IndexMetric::MatrixBatchDistance &batch_distance) {
|
||||
dc_.update_distance(distance, batch_distance);
|
||||
}
|
||||
|
||||
private:
|
||||
void fill_random_to_topk_full(void);
|
||||
|
||||
inline size_t compute_reserve_cnt(uint32_t cur_doc) const {
|
||||
if (cur_doc > kMaxReserveDocCnt) return kMaxReserveDocCnt;
|
||||
if (cur_doc < kMinReserveDocCnt) return kMinReserveDocCnt;
|
||||
return cur_doc;
|
||||
}
|
||||
|
||||
inline uint32_t compute_max_scan_num(uint32_t max_doc_cnt) const {
|
||||
uint32_t max_scan = max_doc_cnt * max_scan_ratio_;
|
||||
if (max_scan < min_scan_limit_) max_scan = min_scan_limit_;
|
||||
if (max_scan > max_scan_limit_) max_scan = max_scan_limit_;
|
||||
return max_scan;
|
||||
}
|
||||
|
||||
constexpr static uint32_t kTriggerReserveCnt = 4096UL;
|
||||
constexpr static uint32_t kMinReserveDocCnt = 4096UL;
|
||||
constexpr static uint32_t kMaxReserveDocCnt = 128 * 1024UL;
|
||||
|
||||
VamanaEntity::Pointer entity_;
|
||||
VamanaDistCalculator dc_;
|
||||
IndexMetric::Pointer metric_;
|
||||
|
||||
bool debug_mode_{false};
|
||||
bool force_padding_topk_{false};
|
||||
uint32_t max_scan_num_{0};
|
||||
uint32_t reserve_max_doc_cnt_{kMinReserveDocCnt};
|
||||
uint32_t topk_{0};
|
||||
uint32_t ef_{VamanaEntity::kDefaultEf};
|
||||
float max_scan_ratio_{VamanaEntity::kDefaultScanRatio};
|
||||
size_t max_scan_limit_{VamanaEntity::kDefaultMaxScanLimit};
|
||||
size_t min_scan_limit_{VamanaEntity::kDefaultMinScanLimit};
|
||||
uint32_t magic_{0U};
|
||||
std::vector<IndexDocumentList> results_{};
|
||||
TopkHeap topk_heap_{};
|
||||
TopkHeap update_heap_{};
|
||||
CandidateHeap candidates_{};
|
||||
VisitFilter visit_filter_{};
|
||||
uint32_t bruteforce_threshold_{};
|
||||
bool fetch_vector_{false};
|
||||
uint32_t type_{kUnknownContext};
|
||||
std::string preprocess_buffer_;
|
||||
|
||||
// Pre-allocated buffers for robust_prune optimization
|
||||
std::vector<const void *> prune_vec_cache_;
|
||||
std::vector<uint8_t> prune_active_;
|
||||
std::vector<float> prune_occlude_factor_;
|
||||
std::vector<std::pair<node_id_t, dist_t>> prune_result_;
|
||||
std::vector<const void *> batch_vecs_buf_;
|
||||
std::vector<float> batch_dists_buf_;
|
||||
std::vector<uint32_t> batch_indices_buf_;
|
||||
|
||||
VisitFilter::Mode filter_mode_{VisitFilter::ByteMap};
|
||||
float filter_negative_prob_{VamanaEntity::kDefaultBFNegativeProbability};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,197 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_meta.h>
|
||||
#include "vamana_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class VamanaDistCalculator {
|
||||
public:
|
||||
typedef std::shared_ptr<VamanaDistCalculator> Pointer;
|
||||
|
||||
VamanaDistCalculator(const VamanaEntity *entity,
|
||||
const IndexMetric::Pointer &metric, uint32_t dim)
|
||||
: entity_(entity),
|
||||
distance_(metric->distance()),
|
||||
batch_distance_(metric->batch_distance()),
|
||||
query_(nullptr),
|
||||
dim_(dim),
|
||||
compare_cnt_(0) {}
|
||||
|
||||
VamanaDistCalculator(const VamanaEntity *entity,
|
||||
const IndexMetric::Pointer &metric, uint32_t dim,
|
||||
const void *query)
|
||||
: entity_(entity),
|
||||
distance_(metric->distance()),
|
||||
batch_distance_(metric->batch_distance()),
|
||||
query_(query),
|
||||
dim_(dim),
|
||||
compare_cnt_(0) {}
|
||||
|
||||
VamanaDistCalculator(const VamanaEntity *entity,
|
||||
const IndexMetric::Pointer &metric)
|
||||
: entity_(entity),
|
||||
distance_(metric->distance()),
|
||||
batch_distance_(metric->batch_distance()),
|
||||
query_(nullptr),
|
||||
dim_(0),
|
||||
compare_cnt_(0) {}
|
||||
|
||||
void update(const VamanaEntity *entity, const IndexMetric::Pointer &metric) {
|
||||
entity_ = entity;
|
||||
distance_ = metric->distance();
|
||||
batch_distance_ = metric->batch_distance();
|
||||
}
|
||||
|
||||
void update(const VamanaEntity *entity, const IndexMetric::Pointer &metric,
|
||||
uint32_t dim) {
|
||||
entity_ = entity;
|
||||
distance_ = metric->distance();
|
||||
batch_distance_ = metric->batch_distance();
|
||||
dim_ = dim;
|
||||
}
|
||||
|
||||
inline void update_distance(
|
||||
const IndexMetric::MatrixDistance &distance,
|
||||
const IndexMetric::MatrixBatchDistance &batch_distance) {
|
||||
distance_ = distance;
|
||||
batch_distance_ = batch_distance;
|
||||
}
|
||||
|
||||
inline void reset_query(const void *query) {
|
||||
error_ = false;
|
||||
query_ = query;
|
||||
}
|
||||
|
||||
inline dist_t dist(const void *vec_lhs, const void *vec_rhs) {
|
||||
if (ailego_unlikely(vec_lhs == nullptr || vec_rhs == nullptr)) {
|
||||
LOG_ERROR("Nullptr of dense vector");
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
float score{0.0f};
|
||||
distance_(vec_lhs, vec_rhs, dim_, &score);
|
||||
return score;
|
||||
}
|
||||
|
||||
inline dist_t dist(const void *vec) {
|
||||
compare_cnt_++;
|
||||
return dist(vec, query_);
|
||||
}
|
||||
|
||||
inline dist_t dist(node_id_t id) {
|
||||
compare_cnt_++;
|
||||
const void *feat = entity_->get_vector(id);
|
||||
if (ailego_unlikely(feat == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, id=%u", id);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
return dist(feat, query_);
|
||||
}
|
||||
|
||||
inline dist_t dist(node_id_t lhs, node_id_t rhs) {
|
||||
compare_cnt_++;
|
||||
const void *feat = entity_->get_vector(lhs);
|
||||
const void *query = entity_->get_vector(rhs);
|
||||
if (ailego_unlikely(feat == nullptr || query == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector");
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
return dist(feat, query);
|
||||
}
|
||||
|
||||
inline void batch_dist(const void **vecs, uint32_t count, float *dists) {
|
||||
compare_cnt_ += count;
|
||||
batch_distance_(vecs, query_, count, dim_, dists);
|
||||
}
|
||||
|
||||
// Single-node batch distance: compute distance between query and a stored
|
||||
// node using batch_distance_. Consistent with HnswDistCalculator::batch_dist.
|
||||
inline dist_t batch_dist(node_id_t id) {
|
||||
compare_cnt_++;
|
||||
const void *feat = entity_->get_vector(id);
|
||||
if (ailego_unlikely(feat == nullptr)) {
|
||||
LOG_ERROR("Get nullptr vector, id=%u", id);
|
||||
error_ = true;
|
||||
return 0.0f;
|
||||
}
|
||||
dist_t score = 0;
|
||||
batch_distance_(&feat, query_, 1, dim_, &score);
|
||||
return score;
|
||||
}
|
||||
|
||||
// Batch distance computation between a base vector and multiple target
|
||||
// vectors. Does NOT use query_ and does NOT increment compare_cnt. Used for
|
||||
// inter-candidate distance computation in robust_prune.
|
||||
//
|
||||
// Uses the single distance function (distance_) in a loop rather than
|
||||
// batch_distance_, because batch_distance_ (turbo AVX512-VNNI) expects
|
||||
// the second argument to be a preprocessed uint8 query (+128 shift),
|
||||
// while base_vec here is a raw int8 stored vector. The single distance
|
||||
// function (AVX2 sign/abs trick) correctly handles two raw int8 inputs.
|
||||
inline void batch_dist_pair(const void *base_vec, const void **vecs,
|
||||
uint32_t count, float *dists) {
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
distance_(base_vec, vecs[i], dim_, &dists[i]);
|
||||
}
|
||||
}
|
||||
|
||||
dist_t operator()(const void *vec) {
|
||||
return dist(vec);
|
||||
}
|
||||
dist_t operator()(node_id_t i) {
|
||||
return dist(i);
|
||||
}
|
||||
dist_t operator()(node_id_t lhs, node_id_t rhs) {
|
||||
return dist(lhs, rhs);
|
||||
}
|
||||
|
||||
inline void clear() {
|
||||
compare_cnt_ = 0;
|
||||
error_ = false;
|
||||
}
|
||||
|
||||
inline void clear_compare_cnt() {
|
||||
compare_cnt_ = 0;
|
||||
}
|
||||
inline bool error() const {
|
||||
return error_;
|
||||
}
|
||||
inline uint32_t compare_cnt() const {
|
||||
return compare_cnt_;
|
||||
}
|
||||
inline uint32_t dimension() const {
|
||||
return dim_;
|
||||
}
|
||||
|
||||
private:
|
||||
VamanaDistCalculator(const VamanaDistCalculator &) = delete;
|
||||
VamanaDistCalculator &operator=(const VamanaDistCalculator &) = delete;
|
||||
|
||||
const VamanaEntity *entity_;
|
||||
IndexMetric::MatrixDistance distance_;
|
||||
IndexMetric::MatrixBatchDistance batch_distance_;
|
||||
const void *query_;
|
||||
uint32_t dim_;
|
||||
uint32_t compare_cnt_;
|
||||
bool error_{false};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,203 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "vamana_entity.h"
|
||||
#include <zvec/ailego/hash/crc32c.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
const std::string VamanaEntity::kGraphHeaderSegmentId = "vamana.graph.header";
|
||||
const std::string VamanaEntity::kGraphFeaturesSegmentId =
|
||||
"vamana.graph.features";
|
||||
const std::string VamanaEntity::kGraphKeysSegmentId = "vamana.graph.keys";
|
||||
const std::string VamanaEntity::kGraphNeighborsSegmentId =
|
||||
"vamana.graph.neighbors";
|
||||
const std::string VamanaEntity::kGraphOffsetsSegmentId = "vamana.graph.offsets";
|
||||
const std::string VamanaEntity::kGraphMappingSegmentId = "vamana.graph.mapping";
|
||||
const std::string VamanaEntity::kGraphNeighborDistsSegmentId =
|
||||
"vamana.graph.neighbor_dists";
|
||||
|
||||
int VamanaEntity::CalcAndAddPadding(const IndexDumper::Pointer &dumper,
|
||||
size_t data_size, size_t *padding_size) {
|
||||
*padding_size = AlignSize(data_size) - data_size;
|
||||
if (*padding_size == 0) {
|
||||
return 0;
|
||||
}
|
||||
std::string padding(*padding_size, '\0');
|
||||
if (dumper->write(padding.data(), *padding_size) != *padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", *padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int64_t VamanaEntity::dump_segment(const IndexDumper::Pointer &dumper,
|
||||
const std::string &segment_id,
|
||||
const void *data, size_t size) const {
|
||||
size_t len = dumper->write(data, size);
|
||||
if (len != size) {
|
||||
LOG_ERROR("Dump segment %s data failed, expect: %lu, actual: %lu",
|
||||
segment_id.c_str(), size, len);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
size_t padding_size = AlignSize(size) - size;
|
||||
if (padding_size > 0) {
|
||||
std::string padding(padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding_size) != padding_size) {
|
||||
LOG_ERROR("Append padding failed, size %lu", padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t crc = ailego::Crc32c::Hash(data, size);
|
||||
int ret = dumper->append(segment_id, size, padding_size, crc);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Dump segment %s meta failed, ret=%d", segment_id.c_str(), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return len + padding_size;
|
||||
}
|
||||
|
||||
int64_t VamanaEntity::dump_header(const IndexDumper::Pointer &dumper,
|
||||
const VamanaHeader &hd) const {
|
||||
return dump_segment(dumper, kGraphHeaderSegmentId, &hd.graph, hd.graph.size);
|
||||
}
|
||||
|
||||
int64_t VamanaEntity::dump_vectors(
|
||||
const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping) const {
|
||||
size_t total_size = doc_cnt() * vector_size();
|
||||
std::vector<uint8_t> buffer(total_size);
|
||||
|
||||
for (node_id_t i = 0; i < doc_cnt(); ++i) {
|
||||
node_id_t old_id = reorder_mapping[i];
|
||||
const void *vec = get_vector(old_id);
|
||||
if (vec == nullptr) {
|
||||
LOG_ERROR("Get vector failed for node %u", old_id);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
memcpy(buffer.data() + static_cast<size_t>(i) * vector_size(), vec,
|
||||
vector_size());
|
||||
}
|
||||
|
||||
return dump_segment(dumper, kGraphFeaturesSegmentId, buffer.data(),
|
||||
total_size);
|
||||
}
|
||||
|
||||
int64_t VamanaEntity::dump_neighbors(
|
||||
const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping,
|
||||
const std::vector<node_id_t> &neighbor_mapping) const {
|
||||
size_t nbr_size = neighbors_size();
|
||||
size_t total_size = doc_cnt() * nbr_size;
|
||||
std::vector<uint8_t> buffer(total_size, 0);
|
||||
|
||||
for (node_id_t i = 0; i < doc_cnt(); ++i) {
|
||||
node_id_t old_id = reorder_mapping[i];
|
||||
const Neighbors nbrs = get_neighbors(old_id);
|
||||
|
||||
auto *hd = reinterpret_cast<NeighborsHeader *>(
|
||||
buffer.data() + static_cast<size_t>(i) * nbr_size);
|
||||
hd->neighbor_cnt = nbrs.size();
|
||||
for (uint32_t j = 0; j < nbrs.size(); ++j) {
|
||||
hd->neighbors[j] = neighbor_mapping[nbrs[j]];
|
||||
}
|
||||
}
|
||||
|
||||
return dump_segment(dumper, kGraphNeighborsSegmentId, buffer.data(),
|
||||
total_size);
|
||||
}
|
||||
|
||||
int64_t VamanaEntity::dump_neighbor_dists(
|
||||
const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping) const {
|
||||
if (!dist_storage_loaded()) {
|
||||
// No distance data to dump — this is fine for read-only indices
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t max_deg = static_cast<uint32_t>(max_degree());
|
||||
size_t dist_entry = max_deg * sizeof(dist_t);
|
||||
size_t total_size = doc_cnt() * dist_entry;
|
||||
std::vector<uint8_t> buffer(total_size, 0);
|
||||
|
||||
for (node_id_t i = 0; i < doc_cnt(); ++i) {
|
||||
node_id_t old_id = reorder_mapping[i];
|
||||
const dist_t *dists = get_neighbor_dists(old_id);
|
||||
if (dists != nullptr) {
|
||||
memcpy(buffer.data() + static_cast<size_t>(i) * dist_entry, dists,
|
||||
dist_entry);
|
||||
}
|
||||
}
|
||||
|
||||
return dump_segment(dumper, kGraphNeighborDistsSegmentId, buffer.data(),
|
||||
total_size);
|
||||
}
|
||||
|
||||
int64_t VamanaEntity::dump_mapping_segment(const IndexDumper::Pointer &dumper,
|
||||
const key_t *keys) const {
|
||||
size_t total_size = doc_cnt() * sizeof(key_t);
|
||||
return dump_segment(dumper, kGraphMappingSegmentId, keys, total_size);
|
||||
}
|
||||
|
||||
void VamanaEntity::reshuffle_vectors(std::vector<node_id_t> *n2o_mapping,
|
||||
std::vector<node_id_t> *o2n_mapping,
|
||||
key_t *keys) const {
|
||||
uint32_t count = doc_cnt();
|
||||
n2o_mapping->resize(count);
|
||||
o2n_mapping->resize(count);
|
||||
|
||||
// Simple identity mapping for now (can be optimized with BFS traversal)
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
(*n2o_mapping)[i] = i;
|
||||
(*o2n_mapping)[i] = i;
|
||||
keys[i] = get_key(i);
|
||||
}
|
||||
}
|
||||
|
||||
int64_t VamanaEntity::dump_segments(const IndexDumper::Pointer &dumper,
|
||||
key_t *keys) const {
|
||||
std::vector<node_id_t> n2o_mapping;
|
||||
std::vector<node_id_t> o2n_mapping;
|
||||
reshuffle_vectors(&n2o_mapping, &o2n_mapping, keys);
|
||||
|
||||
// Remap entry point
|
||||
VamanaHeader dump_header_copy = header_;
|
||||
if (dump_header_copy.graph.entry_point != kInvalidNodeId) {
|
||||
dump_header_copy.graph.entry_point =
|
||||
o2n_mapping[dump_header_copy.graph.entry_point];
|
||||
}
|
||||
|
||||
int64_t hd_size = dump_header(dumper, dump_header_copy);
|
||||
if (hd_size < 0) return hd_size;
|
||||
|
||||
int64_t vec_size = dump_vectors(dumper, n2o_mapping);
|
||||
if (vec_size < 0) return vec_size;
|
||||
|
||||
int64_t nbr_size = dump_neighbors(dumper, n2o_mapping, o2n_mapping);
|
||||
if (nbr_size < 0) return nbr_size;
|
||||
|
||||
int64_t map_size = dump_mapping_segment(dumper, keys);
|
||||
if (map_size < 0) return map_size;
|
||||
|
||||
int64_t dist_size = dump_neighbor_dists(dumper, n2o_mapping);
|
||||
if (dist_size < 0) return dist_size;
|
||||
|
||||
return hd_size + vec_size + nbr_size + map_size + dist_size;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,352 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <string.h>
|
||||
#include <functional>
|
||||
#include <ailego/utility/memory_helper.h>
|
||||
#include <zvec/ailego/container/heap.h>
|
||||
#include <zvec/ailego/logger/logger.h>
|
||||
#include <zvec/core/framework/index_dumper.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/framework/index_storage.h>
|
||||
|
||||
// Reuse typed MemoryBlock and NeighborsT from hnsw_entity.h
|
||||
#include "algorithm/hnsw/hnsw_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
// Vamana graph header — single-layer graph (no hierarchical levels)
|
||||
struct VamanaGraphHeader {
|
||||
uint32_t size;
|
||||
uint32_t version;
|
||||
uint32_t graph_type;
|
||||
uint32_t doc_count;
|
||||
uint32_t vector_size;
|
||||
uint32_t node_size;
|
||||
uint32_t max_degree; // R: maximum out-degree
|
||||
uint32_t search_list_size; // L: search list size for construction
|
||||
uint32_t max_occlusion_size; // C: max candidate size for RobustPrune
|
||||
uint32_t entry_point; // medoid node id
|
||||
uint32_t options;
|
||||
uint32_t reserved_pad;
|
||||
float alpha; // alpha parameter for RobustPrune
|
||||
uint8_t reserved_[4076];
|
||||
};
|
||||
|
||||
static_assert(sizeof(VamanaGraphHeader) % 32 == 0,
|
||||
"VamanaGraphHeader must be aligned with 32 bytes");
|
||||
|
||||
struct VamanaHeader {
|
||||
VamanaHeader() {
|
||||
clear();
|
||||
}
|
||||
|
||||
VamanaHeader(const VamanaHeader &header) {
|
||||
memcpy(this, &header, sizeof(header));
|
||||
}
|
||||
|
||||
VamanaHeader &operator=(const VamanaHeader &header) {
|
||||
memcpy(this, &header, sizeof(header));
|
||||
return *this;
|
||||
}
|
||||
|
||||
void inline reset() {
|
||||
graph.doc_count = 0U;
|
||||
graph.entry_point = kInvalidNodeId;
|
||||
}
|
||||
|
||||
void inline clear() {
|
||||
memset(this, 0, sizeof(VamanaHeader));
|
||||
graph.entry_point = kInvalidNodeId;
|
||||
graph.size = sizeof(VamanaGraphHeader);
|
||||
graph.alpha = 1.2f;
|
||||
}
|
||||
|
||||
size_t max_degree() const {
|
||||
return graph.max_degree;
|
||||
}
|
||||
size_t vector_size() const {
|
||||
return graph.vector_size;
|
||||
}
|
||||
size_t search_list_size() const {
|
||||
return graph.search_list_size;
|
||||
}
|
||||
size_t max_occlusion_size() const {
|
||||
return graph.max_occlusion_size;
|
||||
}
|
||||
float alpha() const {
|
||||
return graph.alpha;
|
||||
}
|
||||
node_id_t entry_point() const {
|
||||
return graph.entry_point;
|
||||
}
|
||||
node_id_t doc_cnt() const {
|
||||
return graph.doc_count;
|
||||
}
|
||||
|
||||
VamanaGraphHeader graph;
|
||||
};
|
||||
|
||||
// VamanaEntity: base class for Vamana graph data management
|
||||
class VamanaEntity {
|
||||
public:
|
||||
VamanaEntity() {}
|
||||
VamanaEntity(const VamanaHeader &hd) {
|
||||
header_ = hd;
|
||||
}
|
||||
virtual ~VamanaEntity() {}
|
||||
|
||||
typedef std::shared_ptr<VamanaEntity> Pointer;
|
||||
|
||||
// Options bit flags (stored in VamanaGraphHeader::options)
|
||||
static constexpr uint32_t kOptionSaturateGraph = 1U << 0;
|
||||
|
||||
// Default constants
|
||||
static constexpr uint32_t kDefaultMaxDegree = 64;
|
||||
static constexpr uint32_t kDefaultSearchListSize = 100;
|
||||
static constexpr uint32_t kDefaultMaxOcclusionSize = 750;
|
||||
static constexpr float kDefaultAlpha = 1.2f;
|
||||
static constexpr bool kDefaultSaturateGraph = false;
|
||||
static constexpr uint32_t kDefaultEf = 200;
|
||||
static constexpr float kDefaultScanRatio = 0.1f;
|
||||
static constexpr uint32_t kDefaultBruteForceThreshold = 1000U;
|
||||
static constexpr uint32_t kDefaultDocsHardLimit = 1 << 30U;
|
||||
static constexpr float kDefaultDocsSoftLimitRatio = 0.9f;
|
||||
static constexpr size_t kMaxChunkSize = 0xFFFFFFFF;
|
||||
static constexpr size_t kDefaultChunkSize = 2UL * 1024UL * 1024UL;
|
||||
static constexpr size_t kDefaultMaxChunkCnt = 50000UL;
|
||||
static constexpr uint32_t kDefaultMinScanLimit = 10000;
|
||||
static constexpr uint32_t kDefaultMaxScanLimit =
|
||||
std::numeric_limits<uint32_t>::max();
|
||||
static constexpr float kDefaultBFNegativeProbability = 0.001f;
|
||||
|
||||
inline size_t max_degree() const {
|
||||
return header_.graph.max_degree;
|
||||
}
|
||||
inline node_id_t *mutable_doc_cnt() {
|
||||
return &header_.graph.doc_count;
|
||||
}
|
||||
inline node_id_t doc_cnt() const {
|
||||
return header_.graph.doc_count;
|
||||
}
|
||||
inline float alpha() const {
|
||||
return header_.graph.alpha;
|
||||
}
|
||||
inline size_t search_list_size() const {
|
||||
return header_.graph.search_list_size;
|
||||
}
|
||||
inline size_t max_occlusion_size() const {
|
||||
return header_.graph.max_occlusion_size;
|
||||
}
|
||||
inline node_id_t entry_point() const {
|
||||
return header_.graph.entry_point;
|
||||
}
|
||||
inline size_t vector_size() const {
|
||||
return header_.graph.vector_size;
|
||||
}
|
||||
inline size_t node_size() const {
|
||||
return header_.graph.node_size;
|
||||
}
|
||||
|
||||
void set_vector_size(size_t size) {
|
||||
header_.graph.vector_size = size;
|
||||
}
|
||||
void set_max_degree(uint32_t val) {
|
||||
header_.graph.max_degree = val;
|
||||
}
|
||||
void set_search_list_size(uint32_t val) {
|
||||
header_.graph.search_list_size = val;
|
||||
}
|
||||
void set_max_occlusion_size(uint32_t val) {
|
||||
header_.graph.max_occlusion_size = val;
|
||||
}
|
||||
void set_alpha(float val) {
|
||||
header_.graph.alpha = val;
|
||||
}
|
||||
|
||||
inline bool saturate_graph() const {
|
||||
return (header_.graph.options & kOptionSaturateGraph) != 0;
|
||||
}
|
||||
void set_saturate_graph(bool val) {
|
||||
if (val) {
|
||||
header_.graph.options |= kOptionSaturateGraph;
|
||||
} else {
|
||||
header_.graph.options &= ~kOptionSaturateGraph;
|
||||
}
|
||||
}
|
||||
|
||||
// Neighbor size: NeighborsHeader + max_degree * sizeof(node_id_t)
|
||||
inline size_t neighbors_size() const {
|
||||
return sizeof(NeighborsHeader) + max_degree() * sizeof(node_id_t);
|
||||
}
|
||||
|
||||
virtual void update_entry_point(node_id_t ep) {
|
||||
header_.graph.entry_point = ep;
|
||||
}
|
||||
|
||||
virtual int cleanup() {
|
||||
header_.clear();
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual const VamanaEntity::Pointer clone() const {
|
||||
return VamanaEntity::Pointer();
|
||||
}
|
||||
|
||||
// Pure virtual interface
|
||||
virtual key_t get_key(node_id_t id) const = 0;
|
||||
virtual const void *get_vector(node_id_t id) const = 0;
|
||||
virtual int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const = 0;
|
||||
virtual int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const = 0;
|
||||
virtual int get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const = 0;
|
||||
virtual const Neighbors get_neighbors(node_id_t id) const = 0;
|
||||
|
||||
virtual int add_vector(key_t key, const void *vec, node_id_t *id) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
virtual int add_vector_with_id(node_id_t id, const void *vec) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
virtual int update_neighbors(
|
||||
node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
virtual void add_neighbor(node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) {}
|
||||
|
||||
// --- Neighbor distance storage (CSR-like, lazy-loaded) ---
|
||||
// Each node has max_degree dist_t slots, the i-th slot stores the distance
|
||||
// from this node to its i-th neighbor. Only allocated/loaded when needed
|
||||
// (first write operation). Search-only paths never touch this data.
|
||||
|
||||
// Ensure distance storage is allocated/loaded. Must be called before
|
||||
// any get/set neighbor dist operations. Thread-safe (idempotent).
|
||||
virtual int ensure_dist_storage() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Whether distance storage is currently loaded
|
||||
virtual bool dist_storage_loaded() const {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get pointer to the distance array for node `id`.
|
||||
// Returns nullptr if dist storage is not loaded.
|
||||
virtual const dist_t *get_neighbor_dists(node_id_t id) const {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Update all neighbor distances for node `id` from a prune result.
|
||||
virtual void update_neighbor_dists(
|
||||
node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {}
|
||||
|
||||
// Set the distance for the `idx`-th neighbor of node `id`.
|
||||
virtual void set_neighbor_dist(node_id_t id, uint32_t idx, dist_t dist) {}
|
||||
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
virtual const void *get_vector_by_key(uint64_t /*key*/) const {
|
||||
return nullptr;
|
||||
}
|
||||
virtual int get_vector_by_key(const key_t /*key*/,
|
||||
IndexStorage::MemoryBlock & /*block*/) const {
|
||||
return IndexError_NotImplemented;
|
||||
}
|
||||
|
||||
static int CalcAndAddPadding(const IndexDumper::Pointer &dumper,
|
||||
size_t data_size, size_t *padding_size);
|
||||
|
||||
protected:
|
||||
inline const VamanaHeader &header() const {
|
||||
return header_;
|
||||
}
|
||||
inline VamanaHeader *mutable_header() {
|
||||
return &header_;
|
||||
}
|
||||
inline size_t header_size() const {
|
||||
return sizeof(header_);
|
||||
}
|
||||
|
||||
void set_node_size(size_t size) {
|
||||
header_.graph.node_size = size;
|
||||
}
|
||||
|
||||
int64_t dump_segments(const IndexDumper::Pointer &dumper, key_t *keys) const;
|
||||
|
||||
int64_t dump_segment(const IndexDumper::Pointer &dumper,
|
||||
const std::string &segment_id, const void *data,
|
||||
size_t size) const;
|
||||
|
||||
int64_t dump_header(const IndexDumper::Pointer &dumper,
|
||||
const VamanaHeader &hd) const;
|
||||
|
||||
int64_t dump_vectors(const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping) const;
|
||||
|
||||
int64_t dump_neighbors(const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping,
|
||||
const std::vector<node_id_t> &neighbor_mapping) const;
|
||||
|
||||
int64_t dump_neighbor_dists(
|
||||
const IndexDumper::Pointer &dumper,
|
||||
const std::vector<node_id_t> &reorder_mapping) const;
|
||||
|
||||
int64_t dump_mapping_segment(const IndexDumper::Pointer &dumper,
|
||||
const key_t *keys) const;
|
||||
|
||||
void reshuffle_vectors(std::vector<node_id_t> *n2o_mapping,
|
||||
std::vector<node_id_t> *o2n_mapping,
|
||||
key_t *keys) const;
|
||||
|
||||
static inline size_t AlignSize(size_t size) {
|
||||
return (size + 0x1F) & (~0x1F);
|
||||
}
|
||||
|
||||
static inline size_t AlignPageSize(size_t size) {
|
||||
size_t page_mask = ailego::MemoryHelper::PageSize() - 1;
|
||||
return (size + page_mask) & (~page_mask);
|
||||
}
|
||||
|
||||
static inline size_t AlignHugePageSize(size_t size) {
|
||||
size_t page_mask = ailego::MemoryHelper::HugePageSize() - 1;
|
||||
return (size + page_mask) & (~page_mask);
|
||||
}
|
||||
|
||||
public:
|
||||
const static std::string kGraphHeaderSegmentId;
|
||||
const static std::string kGraphFeaturesSegmentId;
|
||||
const static std::string kGraphKeysSegmentId;
|
||||
const static std::string kGraphNeighborsSegmentId;
|
||||
const static std::string kGraphOffsetsSegmentId;
|
||||
const static std::string kGraphMappingSegmentId;
|
||||
const static std::string kGraphNeighborDistsSegmentId;
|
||||
|
||||
static constexpr uint32_t kRevision = 0U;
|
||||
|
||||
protected:
|
||||
VamanaHeader header_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,117 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <zvec/core/framework/index_provider.h>
|
||||
#include <zvec/core/framework/index_searcher.h>
|
||||
#include <zvec/core/framework/index_streamer.h>
|
||||
#include "vamana_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class VamanaIndexProvider : public IndexProvider {
|
||||
public:
|
||||
VamanaIndexProvider(const IndexMeta &meta,
|
||||
const VamanaEntity::Pointer &entity,
|
||||
const std::string &owner)
|
||||
: meta_(meta), entity_(entity), owner_class_(owner) {}
|
||||
|
||||
VamanaIndexProvider(const VamanaIndexProvider &) = delete;
|
||||
VamanaIndexProvider &operator=(const VamanaIndexProvider &) = delete;
|
||||
|
||||
public:
|
||||
IndexProvider::Iterator::Pointer create_iterator() override {
|
||||
return VamanaIndexProvider::Iterator::Pointer(new (std::nothrow)
|
||||
Iterator(entity_));
|
||||
}
|
||||
|
||||
size_t count(void) const override {
|
||||
return entity_->doc_cnt();
|
||||
}
|
||||
|
||||
size_t dimension(void) const override {
|
||||
return meta_.dimension();
|
||||
}
|
||||
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return meta_.data_type();
|
||||
}
|
||||
|
||||
size_t element_size(void) const override {
|
||||
return meta_.element_size();
|
||||
}
|
||||
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
return entity_->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
const std::string &owner_class(void) const override {
|
||||
return owner_class_;
|
||||
}
|
||||
|
||||
private:
|
||||
class Iterator : public IndexProvider::Iterator {
|
||||
public:
|
||||
Iterator(const VamanaEntity::Pointer &entity)
|
||||
: entity_(entity), cur_id_(0U) {
|
||||
cur_id_ = get_next_valid_id(0);
|
||||
}
|
||||
|
||||
const void *data(void) const override {
|
||||
return entity_->get_vector(cur_id_);
|
||||
}
|
||||
|
||||
bool is_valid(void) const override {
|
||||
return cur_id_ < entity_->doc_cnt();
|
||||
}
|
||||
|
||||
uint64_t key(void) const override {
|
||||
return entity_->get_key(cur_id_);
|
||||
}
|
||||
|
||||
void next(void) override {
|
||||
cur_id_ = get_next_valid_id(cur_id_ + 1);
|
||||
}
|
||||
|
||||
void reset(void) {
|
||||
cur_id_ = get_next_valid_id(0);
|
||||
}
|
||||
|
||||
private:
|
||||
node_id_t get_next_valid_id(node_id_t start_id) {
|
||||
for (node_id_t i = start_id; i < entity_->doc_cnt(); i++) {
|
||||
if (entity_->get_key(i) != kInvalidNodeId) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return kInvalidNodeId;
|
||||
}
|
||||
|
||||
const VamanaEntity::Pointer entity_;
|
||||
node_id_t cur_id_;
|
||||
};
|
||||
|
||||
const IndexMeta &meta_;
|
||||
const VamanaEntity::Pointer entity_;
|
||||
const std::string owner_class_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,79 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
// Builder parameters
|
||||
static const std::string PARAM_VAMANA_BUILDER_THREAD_COUNT(
|
||||
"proxima.vamana.builder.thread_count");
|
||||
static const std::string PARAM_VAMANA_BUILDER_MEMORY_QUOTA(
|
||||
"proxima.vamana.builder.memory_quota");
|
||||
static const std::string PARAM_VAMANA_BUILDER_MAX_DEGREE(
|
||||
"proxima.vamana.builder.max_degree");
|
||||
static const std::string PARAM_VAMANA_BUILDER_SEARCH_LIST_SIZE(
|
||||
"proxima.vamana.builder.search_list_size");
|
||||
static const std::string PARAM_VAMANA_BUILDER_ALPHA(
|
||||
"proxima.vamana.builder.alpha");
|
||||
static const std::string PARAM_VAMANA_BUILDER_MAX_OCCLUSION_SIZE(
|
||||
"proxima.vamana.builder.max_occlusion_size");
|
||||
|
||||
// Searcher parameters
|
||||
static const std::string PARAM_VAMANA_SEARCHER_SEARCH_LIST_SIZE(
|
||||
"proxima.vamana.searcher.search_list_size");
|
||||
static const std::string PARAM_VAMANA_SEARCHER_BRUTE_FORCE_THRESHOLD(
|
||||
"proxima.vamana.searcher.brute_force_threshold");
|
||||
|
||||
// Streamer parameters
|
||||
static const std::string PARAM_VAMANA_STREAMER_MAX_DEGREE(
|
||||
"proxima.vamana.streamer.max_degree");
|
||||
static const std::string PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE(
|
||||
"proxima.vamana.streamer.search_list_size");
|
||||
static const std::string PARAM_VAMANA_STREAMER_ALPHA(
|
||||
"proxima.vamana.streamer.alpha");
|
||||
static const std::string PARAM_VAMANA_STREAMER_MAX_OCCLUSION_SIZE(
|
||||
"proxima.vamana.streamer.max_occlusion_size");
|
||||
static const std::string PARAM_VAMANA_STREAMER_EF("proxima.vamana.streamer.ef");
|
||||
static const std::string PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD(
|
||||
"proxima.vamana.streamer.brute_force_threshold");
|
||||
static const std::string PARAM_VAMANA_STREAMER_MAX_SCAN_RATIO(
|
||||
"proxima.vamana.streamer.max_scan_ratio");
|
||||
static const std::string PARAM_VAMANA_STREAMER_DOCS_HARD_LIMIT(
|
||||
"proxima.vamana.streamer.docs_hard_limit");
|
||||
static const std::string PARAM_VAMANA_STREAMER_DOCS_SOFT_LIMIT(
|
||||
"proxima.vamana.streamer.docs_soft_limit");
|
||||
static const std::string PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE(
|
||||
"proxima.vamana.streamer.max_index_size");
|
||||
static const std::string PARAM_VAMANA_STREAMER_CHUNK_SIZE(
|
||||
"proxima.vamana.streamer.chunk_size");
|
||||
static const std::string PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE(
|
||||
"proxima.vamana.streamer.get_vector_enable");
|
||||
static const std::string PARAM_VAMANA_STREAMER_FORCE_PADDING_RESULT_ENABLE(
|
||||
"proxima.vamana.streamer.force_padding_result_enable");
|
||||
static const std::string PARAM_VAMANA_STREAMER_USE_ID_MAP(
|
||||
"proxima.vamana.streamer.use_id_map");
|
||||
static const std::string PARAM_VAMANA_STREAMER_MAX_SCAN_LIMIT(
|
||||
"proxima.vamana.streamer.max_scan_limit");
|
||||
static const std::string PARAM_VAMANA_STREAMER_MIN_SCAN_LIMIT(
|
||||
"proxima.vamana.streamer.min_scan_limit");
|
||||
static const std::string PARAM_VAMANA_STREAMER_SATURATE_GRAPH(
|
||||
"proxima.vamana.streamer.saturate_graph");
|
||||
static const std::string PARAM_VAMANA_STREAMER_USE_CONTIGUOUS_MEMORY(
|
||||
"proxima.vamana.streamer.use_contiguous_memory");
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,714 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "vamana_streamer.h"
|
||||
#include <iostream>
|
||||
#include <ailego/pattern/defer.h>
|
||||
#include <ailego/utility/memory_helper.h>
|
||||
#include "vamana_algorithm.h"
|
||||
#include "vamana_context.h"
|
||||
#include "vamana_dist_calculator.h"
|
||||
#include "vamana_index_provider.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
VamanaStreamer::VamanaStreamer() = default;
|
||||
|
||||
VamanaStreamer::~VamanaStreamer() {
|
||||
if (state_ == STATE_INITED) {
|
||||
this->cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
int VamanaStreamer::init(const IndexMeta &imeta, const ailego::Params ¶ms) {
|
||||
meta_ = imeta;
|
||||
meta_.set_streamer("VamanaStreamer", VamanaEntity::kRevision, params);
|
||||
|
||||
params.get(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, &max_index_size_);
|
||||
params.get(PARAM_VAMANA_STREAMER_MAX_DEGREE, &max_degree_);
|
||||
params.get(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, &search_list_size_);
|
||||
params.get(PARAM_VAMANA_STREAMER_ALPHA, &alpha_);
|
||||
params.get(PARAM_VAMANA_STREAMER_MAX_OCCLUSION_SIZE, &max_occlusion_size_);
|
||||
params.get(PARAM_VAMANA_STREAMER_EF, &ef_);
|
||||
params.get(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD,
|
||||
&bruteforce_threshold_);
|
||||
params.get(PARAM_VAMANA_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
|
||||
params.get(PARAM_VAMANA_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
|
||||
params.get(PARAM_VAMANA_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
|
||||
params.get(PARAM_VAMANA_STREAMER_CHUNK_SIZE, &chunk_size_);
|
||||
params.get(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, &get_vector_enabled_);
|
||||
params.get(PARAM_VAMANA_STREAMER_FORCE_PADDING_RESULT_ENABLE,
|
||||
&force_padding_topk_enabled_);
|
||||
params.get(PARAM_VAMANA_STREAMER_USE_ID_MAP, &use_id_map_);
|
||||
params.get(PARAM_VAMANA_STREAMER_DOCS_HARD_LIMIT, &docs_hard_limit_);
|
||||
params.get(PARAM_VAMANA_STREAMER_SATURATE_GRAPH, &saturate_graph_);
|
||||
params.get(PARAM_VAMANA_STREAMER_USE_CONTIGUOUS_MEMORY,
|
||||
&use_contiguous_memory_);
|
||||
|
||||
size_t docs_soft_limit = 0;
|
||||
params.get(PARAM_VAMANA_STREAMER_DOCS_SOFT_LIMIT, &docs_soft_limit);
|
||||
if (docs_soft_limit > 0 && docs_soft_limit > docs_hard_limit_) {
|
||||
LOG_ERROR("[%s] must be >= [%s]",
|
||||
PARAM_VAMANA_STREAMER_DOCS_HARD_LIMIT.c_str(),
|
||||
PARAM_VAMANA_STREAMER_DOCS_SOFT_LIMIT.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
} else if (docs_soft_limit == 0UL) {
|
||||
docs_soft_limit_ =
|
||||
docs_hard_limit_ * VamanaEntity::kDefaultDocsSoftLimitRatio;
|
||||
} else {
|
||||
docs_soft_limit_ = docs_soft_limit;
|
||||
}
|
||||
|
||||
// Validate parameters
|
||||
if (max_degree_ == 0U) max_degree_ = VamanaEntity::kDefaultMaxDegree;
|
||||
if (search_list_size_ == 0U)
|
||||
search_list_size_ = VamanaEntity::kDefaultSearchListSize;
|
||||
if (max_occlusion_size_ == 0U)
|
||||
max_occlusion_size_ = VamanaEntity::kDefaultMaxOcclusionSize;
|
||||
if (alpha_ <= 0.0f) alpha_ = VamanaEntity::kDefaultAlpha;
|
||||
if (ef_ == 0U) ef_ = VamanaEntity::kDefaultEf;
|
||||
if (chunk_size_ == 0UL) chunk_size_ = VamanaEntity::kDefaultChunkSize;
|
||||
if (chunk_size_ > VamanaEntity::kMaxChunkSize) {
|
||||
LOG_ERROR("[%s] must be < %zu", PARAM_VAMANA_STREAMER_CHUNK_SIZE.c_str(),
|
||||
VamanaEntity::kMaxChunkSize);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (max_scan_ratio_ <= 0.0f || max_scan_ratio_ > 1.0f) {
|
||||
LOG_ERROR("[%s] must be in range (0.0f,1.0f]",
|
||||
PARAM_VAMANA_STREAMER_MAX_SCAN_RATIO.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (max_scan_limit_ < min_scan_limit_) {
|
||||
LOG_ERROR("[%s] must be >= [%s]",
|
||||
PARAM_VAMANA_STREAMER_MAX_SCAN_LIMIT.c_str(),
|
||||
PARAM_VAMANA_STREAMER_MIN_SCAN_LIMIT.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
LOG_DEBUG(
|
||||
"Vamana init params: maxIndexSize=%zu docsHardLimit=%zu "
|
||||
"docsSoftLimit=%zu maxDegree=%u searchListSize=%u alpha=%.2f "
|
||||
"maxOcclusionSize=%u ef=%u maxScanRatio=%.3f minScanLimit=%zu "
|
||||
"maxScanLimit=%zu bruteForceThreshold=%zu chunkSize=%zu "
|
||||
"getVectorEnabled=%u forcePadding=%u",
|
||||
max_index_size_, docs_hard_limit_, docs_soft_limit_, max_degree_,
|
||||
search_list_size_, alpha_, max_occlusion_size_, ef_, max_scan_ratio_,
|
||||
min_scan_limit_, max_scan_limit_, bruteforce_threshold_, chunk_size_,
|
||||
get_vector_enabled_, force_padding_topk_enabled_);
|
||||
|
||||
state_ = STATE_INITED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamer::cleanup(void) {
|
||||
if (state_ == STATE_OPENED) {
|
||||
this->close();
|
||||
}
|
||||
|
||||
LOG_INFO("VamanaStreamer cleanup");
|
||||
|
||||
meta_.clear();
|
||||
metric_.reset();
|
||||
stats_.clear();
|
||||
if (entity_) entity_->cleanup();
|
||||
if (alg_) alg_->cleanup();
|
||||
|
||||
max_index_size_ = 0UL;
|
||||
docs_hard_limit_ = VamanaEntity::kDefaultDocsHardLimit;
|
||||
docs_soft_limit_ = 0UL;
|
||||
max_degree_ = VamanaEntity::kDefaultMaxDegree;
|
||||
search_list_size_ = VamanaEntity::kDefaultSearchListSize;
|
||||
max_occlusion_size_ = VamanaEntity::kDefaultMaxOcclusionSize;
|
||||
alpha_ = VamanaEntity::kDefaultAlpha;
|
||||
ef_ = VamanaEntity::kDefaultEf;
|
||||
bruteforce_threshold_ = VamanaEntity::kDefaultBruteForceThreshold;
|
||||
max_scan_limit_ = VamanaEntity::kDefaultMaxScanLimit;
|
||||
min_scan_limit_ = VamanaEntity::kDefaultMinScanLimit;
|
||||
chunk_size_ = VamanaEntity::kDefaultChunkSize;
|
||||
max_scan_ratio_ = VamanaEntity::kDefaultScanRatio;
|
||||
state_ = STATE_INIT;
|
||||
check_crc_enabled_ = false;
|
||||
get_vector_enabled_ = false;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamer::setup_entity() {
|
||||
entity_->set_use_key_info_map(use_id_map_);
|
||||
entity_->set_vector_size(meta_.element_size());
|
||||
entity_->set_chunk_size(chunk_size_);
|
||||
entity_->set_get_vector(get_vector_enabled_);
|
||||
|
||||
// Set Vamana-specific parameters via public setters
|
||||
entity_->set_max_degree(max_degree_);
|
||||
entity_->set_search_list_size(search_list_size_);
|
||||
entity_->set_max_occlusion_size(max_occlusion_size_);
|
||||
entity_->set_alpha(alpha_);
|
||||
entity_->set_saturate_graph(saturate_graph_);
|
||||
|
||||
int ret = entity_->init(docs_hard_limit_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Vamana entity init failed: %s", IndexError::What(ret));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int VamanaStreamer::open(IndexStorage::Pointer stg) {
|
||||
LOG_INFO("VamanaStreamer open");
|
||||
|
||||
if (ailego_unlikely(state_ != STATE_INITED)) {
|
||||
LOG_ERROR("Open storage failed, init streamer first!");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
// Create entity based on storage type
|
||||
switch (stg->memory_block_type()) {
|
||||
case IndexStorage::MemoryBlock::MBT_BUFFERPOOL: {
|
||||
entity_ = std::make_unique<VamanaBufferPoolStreamerEntity>(stats_);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
if (use_contiguous_memory_) {
|
||||
entity_ = std::make_unique<VamanaContiguousStreamerEntity>(stats_);
|
||||
} else {
|
||||
entity_ = std::make_unique<VamanaMmapStreamerEntity>(stats_);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto cleanup_on_error = [this]() {
|
||||
if (entity_) {
|
||||
entity_->close();
|
||||
entity_.reset();
|
||||
}
|
||||
alg_.reset();
|
||||
metric_.reset();
|
||||
};
|
||||
|
||||
int ret = setup_entity();
|
||||
if (ret != 0) {
|
||||
cleanup_on_error();
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = entity_->open(std::move(stg), max_index_size_, check_crc_enabled_);
|
||||
if (ret != 0) {
|
||||
cleanup_on_error();
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Handle IndexMeta
|
||||
IndexMeta index_meta;
|
||||
ret = entity_->get_index_meta(&index_meta);
|
||||
if (ret == IndexError_NoExist) {
|
||||
ret = entity_->set_index_meta(meta_);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to set index meta: %s", IndexError::What(ret));
|
||||
cleanup_on_error();
|
||||
return ret;
|
||||
}
|
||||
} else if (ret != 0) {
|
||||
LOG_ERROR("Failed to get index meta: %s", IndexError::What(ret));
|
||||
cleanup_on_error();
|
||||
return ret;
|
||||
} else {
|
||||
if (index_meta.dimension() != meta_.dimension() ||
|
||||
index_meta.element_size() != meta_.element_size() ||
|
||||
index_meta.metric_name() != meta_.metric_name() ||
|
||||
index_meta.data_type() != meta_.data_type()) {
|
||||
LOG_ERROR("IndexMeta mismatch from the previous in index");
|
||||
cleanup_on_error();
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
auto metric_params = index_meta.metric_params();
|
||||
metric_params.merge(meta_.metric_params());
|
||||
meta_.set_metric(index_meta.metric_name(), 0, metric_params);
|
||||
}
|
||||
|
||||
// Create metric
|
||||
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("Failed to create metric %s", meta_.metric_name().c_str());
|
||||
cleanup_on_error();
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
ret = metric_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to init metric, ret=%d", ret);
|
||||
cleanup_on_error();
|
||||
return ret;
|
||||
}
|
||||
if (!metric_->distance() || !metric_->batch_distance()) {
|
||||
LOG_ERROR("Invalid metric distance functions");
|
||||
cleanup_on_error();
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
add_distance_ = metric_->distance();
|
||||
add_batch_distance_ = metric_->batch_distance();
|
||||
search_distance_ = add_distance_;
|
||||
search_batch_distance_ = add_batch_distance_;
|
||||
|
||||
if (metric_->query_metric() && metric_->query_metric()->distance() &&
|
||||
metric_->query_metric()->batch_distance()) {
|
||||
search_distance_ = metric_->query_metric()->distance();
|
||||
search_batch_distance_ = metric_->query_metric()->batch_distance();
|
||||
}
|
||||
|
||||
// Create algorithm based on entity storage mode
|
||||
switch (entity_->storage_mode()) {
|
||||
case VamanaStorageMode::kBufferPool:
|
||||
alg_ = VamanaAlgorithmBase::UPointer(
|
||||
new VamanaAlgorithm<VamanaBufferPoolStreamerEntity>(
|
||||
static_cast<VamanaBufferPoolStreamerEntity &>(*entity_)));
|
||||
break;
|
||||
case VamanaStorageMode::kContiguous: {
|
||||
auto &contiguous_entity =
|
||||
static_cast<VamanaContiguousStreamerEntity &>(*entity_);
|
||||
int build_ret = contiguous_entity.build_contiguous_memory();
|
||||
if (build_ret != 0) {
|
||||
LOG_ERROR("Failed to build contiguous memory, ret=%d", build_ret);
|
||||
cleanup_on_error();
|
||||
return build_ret;
|
||||
}
|
||||
alg_ = VamanaAlgorithmBase::UPointer(
|
||||
new VamanaAlgorithm<VamanaContiguousStreamerEntity>(
|
||||
contiguous_entity));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
alg_ = VamanaAlgorithmBase::UPointer(
|
||||
new VamanaAlgorithm<VamanaMmapStreamerEntity>(
|
||||
static_cast<VamanaMmapStreamerEntity &>(*entity_)));
|
||||
break;
|
||||
}
|
||||
|
||||
ret = alg_->init();
|
||||
if (ret != 0) {
|
||||
cleanup_on_error();
|
||||
return ret;
|
||||
}
|
||||
|
||||
state_ = STATE_OPENED;
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamer::close(void) {
|
||||
LOG_INFO("VamanaStreamer close");
|
||||
|
||||
stats_.clear();
|
||||
meta_.set_metric(metric_->name(), 0, metric_->params());
|
||||
entity_->set_index_meta(meta_);
|
||||
int ret = entity_->close();
|
||||
if (ret != 0) return ret;
|
||||
state_ = STATE_INITED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamer::flush(uint64_t checkpoint) {
|
||||
LOG_INFO("VamanaStreamer flush checkpoint=%zu", (size_t)checkpoint);
|
||||
|
||||
meta_.set_metric(metric_->name(), 0, metric_->params());
|
||||
entity_->set_index_meta(meta_);
|
||||
return entity_->flush(checkpoint);
|
||||
}
|
||||
|
||||
int VamanaStreamer::dump(const IndexDumper::Pointer &dumper) {
|
||||
LOG_INFO("VamanaStreamer dump");
|
||||
|
||||
shared_mutex_.lock();
|
||||
AILEGO_DEFER([&]() { shared_mutex_.unlock(); });
|
||||
|
||||
meta_.set_searcher("VamanaSearcher", VamanaEntity::kRevision,
|
||||
ailego::Params());
|
||||
|
||||
int ret = IndexHelper::SerializeToDumper(meta_, dumper.get());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to serialize meta into dumper.");
|
||||
return ret;
|
||||
}
|
||||
return entity_->dump(dumper);
|
||||
}
|
||||
|
||||
IndexStreamer::Context::Pointer VamanaStreamer::create_context(void) const {
|
||||
if (ailego_unlikely(state_ != STATE_OPENED)) {
|
||||
LOG_ERROR("Create context failed, open storage first!");
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
VamanaEntity::Pointer entity = entity_->clone();
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("CreateContext clone failed");
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
VamanaContext *ctx =
|
||||
new (std::nothrow) VamanaContext(meta_.dimension(), metric_, entity);
|
||||
if (ailego_unlikely(ctx == nullptr)) {
|
||||
LOG_ERROR("Failed to new VamanaContext");
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
ctx->set_ef(ef_);
|
||||
ctx->set_max_scan_limit(max_scan_limit_);
|
||||
ctx->set_min_scan_limit(min_scan_limit_);
|
||||
ctx->set_max_scan_ratio(max_scan_ratio_);
|
||||
ctx->set_magic(magic_);
|
||||
ctx->set_force_padding_topk(force_padding_topk_enabled_);
|
||||
ctx->set_bruteforce_threshold(bruteforce_threshold_);
|
||||
|
||||
if (ailego_unlikely(ctx->init(VamanaContext::kStreamerContext) != 0)) {
|
||||
LOG_ERROR("Init VamanaContext failed");
|
||||
delete ctx;
|
||||
return Context::Pointer();
|
||||
}
|
||||
|
||||
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
|
||||
|
||||
return Context::Pointer(ctx);
|
||||
}
|
||||
|
||||
IndexProvider::Pointer VamanaStreamer::create_provider(void) const {
|
||||
LOG_DEBUG("VamanaStreamer create provider");
|
||||
|
||||
auto entity = entity_->clone();
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("Clone VamanaEntity failed");
|
||||
return nullptr;
|
||||
}
|
||||
return IndexProvider::Pointer(
|
||||
new VamanaIndexProvider(meta_, entity, "VamanaStreamer"));
|
||||
}
|
||||
|
||||
int VamanaStreamer::update_context(VamanaContext *ctx) const {
|
||||
const VamanaEntity::Pointer entity = entity_->clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone search context entity");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
ctx->set_max_scan_limit(max_scan_limit_);
|
||||
ctx->set_min_scan_limit(min_scan_limit_);
|
||||
ctx->set_max_scan_ratio(max_scan_ratio_);
|
||||
ctx->set_bruteforce_threshold(bruteforce_threshold_);
|
||||
return ctx->update_context(VamanaContext::kStreamerContext, meta_, metric_,
|
||||
entity, magic_);
|
||||
}
|
||||
|
||||
int VamanaStreamer::add_impl(uint64_t pkey, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) {
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) return ret;
|
||||
|
||||
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to VamanaContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) return ret;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(entity_->doc_cnt() >= docs_soft_limit_)) {
|
||||
if (entity_->doc_cnt() >= docs_hard_limit_) {
|
||||
LOG_ERROR("Current docs %u exceed hard limit", entity_->doc_cnt());
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_IndexFull;
|
||||
} else {
|
||||
LOG_WARN("Current docs %u exceed soft limit", entity_->doc_cnt());
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
|
||||
LOG_ERROR("Cannot add vector while dumping index");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
|
||||
ctx->reset_query(query);
|
||||
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
|
||||
|
||||
if (metric_->support_train()) {
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
ret = metric_->train(query, meta_.dimension());
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Vamana streamer metric train failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
node_id_t id;
|
||||
ret = entity_->add_vector(pkey, query, &id);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Vamana streamer add vector failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = alg_->add_node(id, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Vamana streamer add node failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
(*stats_.mutable_added_count())++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamer::add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) {
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) return ret;
|
||||
|
||||
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to VamanaContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) return ret;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(entity_->doc_cnt() >= docs_soft_limit_)) {
|
||||
if (entity_->doc_cnt() >= docs_hard_limit_) {
|
||||
LOG_ERROR("Current docs %u exceed hard limit", entity_->doc_cnt());
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_IndexFull;
|
||||
} else {
|
||||
LOG_WARN("Current docs %u exceed soft limit", entity_->doc_cnt());
|
||||
}
|
||||
}
|
||||
|
||||
if (ailego_unlikely(!shared_mutex_.try_lock_shared())) {
|
||||
LOG_ERROR("Cannot add vector while dumping index");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
AILEGO_DEFER([&]() { shared_mutex_.unlock_shared(); });
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(add_distance_, add_batch_distance_);
|
||||
ctx->reset_query(query);
|
||||
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
|
||||
|
||||
if (metric_->support_train()) {
|
||||
const std::lock_guard<std::mutex> lk(mutex_);
|
||||
ret = metric_->train(query, meta_.dimension());
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Vamana streamer metric train failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
ret = entity_->add_vector_with_id(id, query);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Vamana streamer add vector failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = alg_->add_node(id, ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Vamana streamer add node failed");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) {
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
(*stats_.mutable_added_count())++;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const {
|
||||
return search_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
int VamanaStreamer::search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) return ret;
|
||||
|
||||
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to VamanaContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
if (entity_->doc_cnt() <= ctx->get_bruteforce_threshold()) {
|
||||
return search_bf_impl(query, qmeta, count, context);
|
||||
}
|
||||
|
||||
if (ctx->magic() != magic_) {
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) return ret;
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
|
||||
ctx->resize_results(count);
|
||||
ctx->check_need_adjuct_ctx(entity_->doc_cnt());
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
ctx->reset_query(query);
|
||||
ret = alg_->search(ctx);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Vamana search failed");
|
||||
return ret;
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) return IndexError_Runtime;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void VamanaStreamer::print_debug_info() {
|
||||
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
|
||||
if (entity_->get_key(id) == kInvalidKey) continue;
|
||||
Neighbors neighbours = entity_->get_neighbors(id);
|
||||
std::cout << "node: " << id << "; ";
|
||||
if (neighbours.size() == 0) {
|
||||
std::cout << std::endl;
|
||||
continue;
|
||||
}
|
||||
for (uint32_t i = 0; i < neighbours.size(); ++i) {
|
||||
std::cout << neighbours[i];
|
||||
if (i == neighbours.size() - 1) {
|
||||
std::cout << std::endl;
|
||||
} else {
|
||||
std::cout << ", ";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int VamanaStreamer::search_bf_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
int VamanaStreamer::search_bf_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) return ret;
|
||||
|
||||
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to VamanaContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) return ret;
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
|
||||
ctx->resize_results(count);
|
||||
|
||||
const auto &filter = static_cast<IndexContext *>(ctx)->filter();
|
||||
auto &topk = ctx->topk_heap();
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
ctx->reset_query(query);
|
||||
topk.clear();
|
||||
for (node_id_t id = 0; id < entity_->doc_cnt(); ++id) {
|
||||
if (entity_->get_key(id) == kInvalidKey) continue;
|
||||
if (!filter.is_valid() || !filter(entity_->get_key(id))) {
|
||||
dist_t dist = ctx->dist_calculator().batch_dist(id);
|
||||
topk.emplace(id, dist);
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) return IndexError_Runtime;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamer::search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
int ret = check_params(query, qmeta);
|
||||
if (ailego_unlikely(ret != 0)) return ret;
|
||||
|
||||
VamanaContext *ctx = dynamic_cast<VamanaContext *>(context.get());
|
||||
ailego_do_if_false(ctx) {
|
||||
LOG_ERROR("Cast context to VamanaContext failed");
|
||||
return IndexError_Cast;
|
||||
}
|
||||
if (ctx->magic() != magic_) {
|
||||
ret = update_context(ctx);
|
||||
if (ret != 0) return ret;
|
||||
}
|
||||
|
||||
ctx->clear();
|
||||
ctx->update_dist_caculator_distance(search_distance_, search_batch_distance_);
|
||||
ctx->resize_results(count);
|
||||
|
||||
auto &topk = ctx->topk_heap();
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
ctx->reset_query(query);
|
||||
topk.clear();
|
||||
for (const auto &keys : p_keys) {
|
||||
for (auto key : keys) {
|
||||
node_id_t id = entity_->get_id(key);
|
||||
if (id == kInvalidNodeId) continue;
|
||||
dist_t dist = ctx->dist_calculator().batch_dist(id);
|
||||
topk.emplace(id, dist);
|
||||
}
|
||||
}
|
||||
ctx->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
|
||||
if (ailego_unlikely(ctx->error())) return IndexError_Runtime;
|
||||
return 0;
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_STREAMER(VamanaStreamer);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,192 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "vamana_algorithm.h"
|
||||
#include "vamana_streamer_entity.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class VamanaStreamer : public IndexStreamer {
|
||||
public:
|
||||
using ContextPointer = IndexStreamer::Context::Pointer;
|
||||
|
||||
VamanaStreamer(void);
|
||||
virtual ~VamanaStreamer(void);
|
||||
|
||||
VamanaStreamer(const VamanaStreamer &) = delete;
|
||||
VamanaStreamer &operator=(const VamanaStreamer &) = delete;
|
||||
|
||||
protected:
|
||||
virtual int init(const IndexMeta &imeta,
|
||||
const ailego::Params ¶ms) override;
|
||||
|
||||
virtual int cleanup(void) override;
|
||||
|
||||
virtual Context::Pointer create_context(void) const override;
|
||||
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
virtual int add_impl(uint64_t pkey, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
virtual int add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
return entity_->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
virtual int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
virtual const void *get_vector_by_id(uint32_t id) const override {
|
||||
return entity_->get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_id(
|
||||
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector(id, block);
|
||||
}
|
||||
|
||||
virtual int open(IndexStorage::Pointer stg) override;
|
||||
|
||||
virtual int close(void) override;
|
||||
|
||||
virtual int flush(uint64_t checkpoint) override;
|
||||
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
virtual const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
virtual void print_debug_info() override;
|
||||
|
||||
private:
|
||||
inline int check_params(const void *query,
|
||||
const IndexQueryMeta &qmeta) const {
|
||||
if (ailego_unlikely(!query)) {
|
||||
LOG_ERROR("null query");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (ailego_unlikely(qmeta.dimension() != meta_.dimension() ||
|
||||
qmeta.data_type() != meta_.data_type() ||
|
||||
qmeta.element_size() != meta_.element_size())) {
|
||||
LOG_ERROR("Unsupported query meta");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int setup_entity();
|
||||
int update_context(VamanaContext *ctx) const;
|
||||
|
||||
private:
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_OPENED = 2 };
|
||||
|
||||
class Stats : public IndexStreamer::Stats {
|
||||
public:
|
||||
void clear(void) {
|
||||
set_revision_id(0u);
|
||||
set_loaded_count(0u);
|
||||
set_added_count(0u);
|
||||
set_discarded_count(0u);
|
||||
set_index_size(0u);
|
||||
set_dumped_size(0u);
|
||||
set_check_point(0u);
|
||||
set_create_time(0u);
|
||||
set_update_time(0u);
|
||||
clear_attributes();
|
||||
}
|
||||
};
|
||||
|
||||
std::unique_ptr<VamanaStreamerEntity> entity_;
|
||||
VamanaAlgorithmBase::UPointer alg_;
|
||||
IndexMeta meta_{};
|
||||
IndexMetric::Pointer metric_{};
|
||||
|
||||
IndexMetric::MatrixDistance add_distance_{};
|
||||
IndexMetric::MatrixDistance search_distance_{};
|
||||
IndexMetric::MatrixBatchDistance add_batch_distance_{};
|
||||
IndexMetric::MatrixBatchDistance search_batch_distance_{};
|
||||
|
||||
Stats stats_{};
|
||||
std::mutex mutex_{};
|
||||
|
||||
size_t max_index_size_{0UL};
|
||||
size_t chunk_size_{VamanaEntity::kDefaultChunkSize};
|
||||
size_t docs_hard_limit_{VamanaEntity::kDefaultDocsHardLimit};
|
||||
size_t docs_soft_limit_{0UL};
|
||||
uint32_t max_degree_{VamanaEntity::kDefaultMaxDegree};
|
||||
uint32_t search_list_size_{VamanaEntity::kDefaultSearchListSize};
|
||||
uint32_t max_occlusion_size_{VamanaEntity::kDefaultMaxOcclusionSize};
|
||||
float alpha_{VamanaEntity::kDefaultAlpha};
|
||||
uint32_t ef_{VamanaEntity::kDefaultEf};
|
||||
size_t bruteforce_threshold_{VamanaEntity::kDefaultBruteForceThreshold};
|
||||
size_t max_scan_limit_{VamanaEntity::kDefaultMaxScanLimit};
|
||||
size_t min_scan_limit_{VamanaEntity::kDefaultMinScanLimit};
|
||||
float bf_negative_prob_{VamanaEntity::kDefaultBFNegativeProbability};
|
||||
float max_scan_ratio_{VamanaEntity::kDefaultScanRatio};
|
||||
|
||||
uint32_t magic_{0U};
|
||||
State state_{STATE_INIT};
|
||||
bool check_crc_enabled_{false};
|
||||
bool get_vector_enabled_{false};
|
||||
bool force_padding_topk_enabled_{false};
|
||||
bool use_id_map_{true};
|
||||
bool saturate_graph_{VamanaEntity::kDefaultSaturateGraph};
|
||||
bool use_contiguous_memory_{false};
|
||||
|
||||
ailego::SharedMutex shared_mutex_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,773 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "vamana_streamer_entity.h"
|
||||
#if defined(__linux__) || defined(__APPLE__)
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#include <ailego/utility/memory_helper.h>
|
||||
#include <zvec/ailego/hash/crc32c.h>
|
||||
#include <zvec/core/framework/index_stats.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
VamanaStreamerEntity::VamanaStreamerEntity(IndexStreamer::Stats &stats)
|
||||
: stats_(stats) {
|
||||
keys_map_lock_ = std::make_shared<ailego::SharedMutex>();
|
||||
keys_map_ = std::make_shared<HashMap<key_t, node_id_t>>();
|
||||
keys_map_->set_empty_key(kInvalidKey);
|
||||
broker_ = std::make_shared<ChunkBroker>(stats);
|
||||
}
|
||||
|
||||
VamanaStreamerEntity::~VamanaStreamerEntity() {}
|
||||
|
||||
int VamanaStreamerEntity::cleanup() {
|
||||
node_chunks_.clear();
|
||||
if (keys_map_) {
|
||||
keys_map_->clear();
|
||||
}
|
||||
header_.clear();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::init(size_t /*max_doc_cnt*/) {
|
||||
// node_size = vector_size + key_size + neighbors_size
|
||||
set_node_size(vector_size() + sizeof(key_t) + neighbors_size());
|
||||
neighbor_size_ = neighbors_size();
|
||||
return 0;
|
||||
}
|
||||
|
||||
key_t VamanaStreamerEntity::get_key(node_id_t id) const {
|
||||
if (!use_key_info_map_) return id;
|
||||
auto loc = get_key_chunk_loc(id);
|
||||
if (ailego_unlikely(loc.first >= node_chunks_.size())) return kInvalidKey;
|
||||
const void *ptr = nullptr;
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, sizeof(key_t));
|
||||
if (ailego_unlikely(ret != sizeof(key_t))) {
|
||||
LOG_ERROR("Read key failed, ret=%zu", ret);
|
||||
return kInvalidKey;
|
||||
}
|
||||
return *reinterpret_cast<const key_t *>(ptr);
|
||||
}
|
||||
|
||||
const void *VamanaStreamerEntity::get_vector(node_id_t id) const {
|
||||
auto loc = get_vector_chunk_loc(id);
|
||||
if (ailego_unlikely(loc.first >= node_chunks_.size())) return nullptr;
|
||||
const void *ptr = nullptr;
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, vector_size());
|
||||
if (ailego_unlikely(ret != vector_size())) {
|
||||
LOG_ERROR("Read vector failed, ret=%zu", ret);
|
||||
return nullptr;
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const {
|
||||
auto loc = get_vector_chunk_loc(id);
|
||||
if (ailego_unlikely(loc.first >= node_chunks_.size()))
|
||||
return IndexError_NoExist;
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, block, vector_size());
|
||||
if (ailego_unlikely(ret != vector_size())) {
|
||||
LOG_ERROR("Read vector failed, ret=%zu", ret);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const {
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
vecs[i] = get_vector(ids[i]);
|
||||
if (ailego_unlikely(vecs[i] == nullptr)) {
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const {
|
||||
vec_blocks.resize(count);
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
int ret = get_vector(ids[i], vec_blocks[i]);
|
||||
if (ailego_unlikely(ret != 0)) return ret;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const Neighbors VamanaStreamerEntity::get_neighbors(node_id_t id) const {
|
||||
auto loc = get_neighbor_chunk_loc(id);
|
||||
IndexStorage::MemoryBlock mem_block;
|
||||
size_t ret = loc.first->read(loc.second, mem_block, neighbor_size_);
|
||||
if (ailego_unlikely(ret != neighbor_size_)) {
|
||||
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
|
||||
return Neighbors();
|
||||
}
|
||||
return Neighbors(mem_block);
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::add_vector(key_t key, const void *vec,
|
||||
node_id_t *id) {
|
||||
Chunk::Pointer node_chunk;
|
||||
size_t chunk_offset = static_cast<size_t>(-1);
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
|
||||
node_id_t local_id = static_cast<node_id_t>(doc_cnt());
|
||||
uint32_t chunk_index = node_chunks_.size() - 1U;
|
||||
if (chunk_index == static_cast<uint32_t>(-1) ||
|
||||
(node_chunks_[chunk_index]->data_size() >=
|
||||
node_cnt_per_chunk_ * node_size())) {
|
||||
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
|
||||
LOG_ERROR("add vector failed for no memory quota");
|
||||
return IndexError_IndexFull;
|
||||
}
|
||||
chunk_index++;
|
||||
if (auto dret = ensure_dist_chunk_for(chunk_index); dret != 0) {
|
||||
return dret;
|
||||
}
|
||||
auto p = broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_NODE, chunk_index,
|
||||
chunk_size_);
|
||||
if (ailego_unlikely(p.first != 0)) {
|
||||
LOG_ERROR("Alloc data chunk failed");
|
||||
return p.first;
|
||||
}
|
||||
node_chunk = p.second;
|
||||
chunk_offset = 0UL;
|
||||
node_chunks_.emplace_back(node_chunk);
|
||||
} else {
|
||||
node_chunk = node_chunks_[chunk_index];
|
||||
chunk_offset = node_chunk->data_size();
|
||||
}
|
||||
|
||||
// Write vector
|
||||
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
|
||||
if (ailego_unlikely(size != vector_size())) {
|
||||
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
// Write key
|
||||
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
|
||||
if (ailego_unlikely(size != sizeof(key_t))) {
|
||||
LOG_ERROR("Chunk write key failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
// Neighbors are initialized to zero by default (chunk is zero-filled)
|
||||
|
||||
chunk_offset += node_size();
|
||||
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
|
||||
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
if (use_key_info_map_) {
|
||||
keys_map_lock_->lock();
|
||||
(*keys_map_)[key] = local_id;
|
||||
keys_map_lock_->unlock();
|
||||
}
|
||||
|
||||
*mutable_doc_cnt() += 1;
|
||||
broker_->mark_dirty();
|
||||
*id = local_id;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::add_vector_with_id(node_id_t id, const void *vec) {
|
||||
Chunk::Pointer node_chunk;
|
||||
size_t chunk_offset = static_cast<size_t>(-1);
|
||||
key_t key = id;
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
|
||||
auto func_get_node_chunk_and_offset = [&](node_id_t node_id) -> int {
|
||||
uint32_t chunk_idx = node_id >> node_index_mask_bits_;
|
||||
ailego_assert_with(chunk_idx <= node_chunks_.size(), "invalid chunk idx");
|
||||
if (chunk_idx == node_chunks_.size()) {
|
||||
if (ailego_unlikely(node_chunks_.capacity() == node_chunks_.size())) {
|
||||
LOG_ERROR("add vector failed for no memory quota");
|
||||
return IndexError_IndexFull;
|
||||
}
|
||||
if (auto dret = ensure_dist_chunk_for(chunk_idx); dret != 0) {
|
||||
return dret;
|
||||
}
|
||||
auto p = broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx,
|
||||
chunk_size_);
|
||||
if (ailego_unlikely(p.first != 0)) {
|
||||
LOG_ERROR("Alloc data chunk failed");
|
||||
return p.first;
|
||||
}
|
||||
node_chunk = p.second;
|
||||
node_chunks_.emplace_back(node_chunk);
|
||||
}
|
||||
node_chunk = node_chunks_[chunk_idx];
|
||||
chunk_offset = (node_id & node_index_mask_) * node_size();
|
||||
return 0;
|
||||
};
|
||||
|
||||
// Fill gaps with invalid keys
|
||||
for (size_t start_id = doc_cnt(); start_id < id; ++start_id) {
|
||||
if (auto ret = func_get_node_chunk_and_offset(start_id); ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
size_t size = node_chunk->write(chunk_offset + vector_size(), &kInvalidKey,
|
||||
sizeof(key_t));
|
||||
if (ailego_unlikely(size != sizeof(key_t))) {
|
||||
LOG_ERROR("Chunk write key failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
chunk_offset += node_size();
|
||||
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
|
||||
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
|
||||
if (auto ret = func_get_node_chunk_and_offset(id); ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Write vector
|
||||
size_t size = node_chunk->write(chunk_offset, vec, vector_size());
|
||||
if (ailego_unlikely(size != vector_size())) {
|
||||
LOG_ERROR("Chunk write vec failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
// Write key
|
||||
size = node_chunk->write(chunk_offset + vector_size(), &key, sizeof(key_t));
|
||||
if (ailego_unlikely(size != sizeof(key_t))) {
|
||||
LOG_ERROR("Chunk write key failed, ret=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
if (*mutable_doc_cnt() <= id) {
|
||||
*mutable_doc_cnt() = id + 1;
|
||||
chunk_offset += node_size();
|
||||
if (ailego_unlikely(node_chunk->resize(chunk_offset) != chunk_offset)) {
|
||||
LOG_ERROR("Chunk resize to %zu failed", chunk_offset);
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
|
||||
if (use_key_info_map_) {
|
||||
keys_map_lock_->lock();
|
||||
(*keys_map_)[key] = id;
|
||||
keys_map_lock_->unlock();
|
||||
}
|
||||
|
||||
broker_->mark_dirty();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::update_neighbors(
|
||||
node_id_t id, const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
|
||||
auto loc = get_neighbor_chunk_loc(id);
|
||||
uint32_t count = std::min(static_cast<uint32_t>(neighbors.size()),
|
||||
static_cast<uint32_t>(max_degree()));
|
||||
|
||||
// Build neighbor data in a local buffer
|
||||
size_t nbr_size = neighbors_size();
|
||||
std::vector<uint8_t> buffer(nbr_size, 0);
|
||||
auto *hd = reinterpret_cast<NeighborsHeader *>(buffer.data());
|
||||
hd->neighbor_cnt = count;
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
hd->neighbors[i] = neighbors[i].first;
|
||||
}
|
||||
|
||||
size_t ret = loc.first->write(loc.second, buffer.data(), nbr_size);
|
||||
if (ailego_unlikely(ret != nbr_size)) {
|
||||
LOG_ERROR("Write neighbors failed, ret=%zu", ret);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void VamanaStreamerEntity::add_neighbor(node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) {
|
||||
auto loc = get_neighbor_chunk_loc(id);
|
||||
if (size >= max_degree()) return;
|
||||
|
||||
// Read current neighbors
|
||||
IndexStorage::MemoryBlock mem_block;
|
||||
size_t ret = loc.first->read(loc.second, mem_block, neighbor_size_);
|
||||
if (ailego_unlikely(ret != neighbor_size_)) {
|
||||
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
|
||||
return;
|
||||
}
|
||||
|
||||
// Copy to mutable buffer, update, and write back
|
||||
std::vector<uint8_t> buffer(neighbor_size_);
|
||||
memcpy(buffer.data(), mem_block.data(), neighbor_size_);
|
||||
auto *hd = reinterpret_cast<NeighborsHeader *>(buffer.data());
|
||||
hd->neighbors[size] = neighbor_id;
|
||||
hd->neighbor_cnt = size + 1;
|
||||
|
||||
ret = loc.first->write(loc.second, buffer.data(), neighbor_size_);
|
||||
if (ailego_unlikely(ret != neighbor_size_)) {
|
||||
LOG_ERROR("Write neighbor failed, ret=%zu", ret);
|
||||
}
|
||||
}
|
||||
|
||||
void VamanaStreamerEntity::update_entry_point(node_id_t ep) {
|
||||
VamanaEntity::update_entry_point(ep);
|
||||
flush_header();
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::open(IndexStorage::Pointer stg,
|
||||
uint64_t max_index_size, bool check_crc) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
bool huge_page = stg->isHugePage();
|
||||
|
||||
int ret = broker_->open(std::move(stg), chunk_size_, check_crc);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Open index failed: %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = init_chunk_params(max_index_size, huge_page);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("init_chunk_params failed: %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
broker_->set_max_chunks_size(max_index_size_);
|
||||
|
||||
// Init header
|
||||
auto header_chunk = broker_->get_chunk(ChunkBroker::CHUNK_TYPE_HEADER,
|
||||
ChunkBroker::kDefaultChunkSeqId);
|
||||
if (!header_chunk) {
|
||||
// Open empty index, create header
|
||||
auto p =
|
||||
broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_HEADER,
|
||||
ChunkBroker::kDefaultChunkSeqId, header_size());
|
||||
if (ailego_unlikely(p.first != 0)) {
|
||||
LOG_ERROR("Alloc header chunk failed");
|
||||
return p.first;
|
||||
}
|
||||
size_t size = p.second->write(0UL, &header(), header_size());
|
||||
if (ailego_unlikely(size != header_size())) {
|
||||
LOG_ERROR("Write header chunk failed");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Open existing index
|
||||
ret = init_chunks(header_chunk);
|
||||
if (ailego_unlikely(ret != 0)) return ret;
|
||||
|
||||
// Verify total docs
|
||||
node_id_t total_vecs = 0;
|
||||
if (!node_chunks_.empty()) {
|
||||
size_t last_idx = node_chunks_.size() - 1;
|
||||
if (node_chunks_[last_idx]->data_size() % node_size()) {
|
||||
LOG_WARN("The index may be broken");
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
total_vecs = last_idx * node_cnt_per_chunk_ +
|
||||
node_chunks_[last_idx]->data_size() / node_size();
|
||||
}
|
||||
|
||||
LOG_INFO("Open Vamana index, maxDegree=%zu docCnt=%u totalVecs=%u",
|
||||
max_degree(), doc_cnt(), total_vecs);
|
||||
|
||||
if (doc_cnt() != total_vecs) {
|
||||
LOG_WARN("Index closed abnormally, using totalVecs as curDocCnt");
|
||||
*mutable_doc_cnt() = total_vecs;
|
||||
}
|
||||
|
||||
// Rebuild key map
|
||||
if (use_key_info_map_) {
|
||||
for (node_id_t i = 0; i < doc_cnt(); ++i) {
|
||||
key_t k = get_key(i);
|
||||
if (k != kInvalidKey) {
|
||||
(*keys_map_)[k] = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stats_.set_loaded_count(doc_cnt());
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::init_chunks(const Chunk::Pointer &header_chunk) {
|
||||
// Read header from chunk
|
||||
const void *hd_ptr = nullptr;
|
||||
size_t ret = header_chunk->read(0UL, &hd_ptr, header_size());
|
||||
if (ailego_unlikely(ret != header_size())) {
|
||||
LOG_ERROR("Read header chunk failed");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
auto *hd = reinterpret_cast<const VamanaHeader *>(hd_ptr);
|
||||
|
||||
// Validate
|
||||
if (vector_size() != hd->vector_size()) {
|
||||
LOG_ERROR("vector size %zu mismatch index previous %zu", vector_size(),
|
||||
hd->vector_size());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
if (max_degree() != hd->max_degree()) {
|
||||
LOG_ERROR("max_degree %zu mismatch index previous %zu", max_degree(),
|
||||
hd->max_degree());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
*mutable_header() = *hd;
|
||||
|
||||
// Load node chunks
|
||||
size_t chunk_cnt = broker_->get_chunk_cnt(ChunkBroker::CHUNK_TYPE_NODE);
|
||||
for (size_t i = 0; i < chunk_cnt; ++i) {
|
||||
auto chunk = broker_->get_chunk(ChunkBroker::CHUNK_TYPE_NODE, i);
|
||||
if (ailego_unlikely(!chunk)) {
|
||||
LOG_ERROR("Get node chunk %zu failed", i);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
node_chunks_.emplace_back(std::move(chunk));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::close() {
|
||||
LOG_DEBUG("close Vamana index");
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
flush_header();
|
||||
mutable_header()->reset();
|
||||
keys_map_->clear();
|
||||
header_.clear();
|
||||
node_chunks_.clear();
|
||||
dist_chunks_.clear();
|
||||
dist_loaded_ = false;
|
||||
return broker_->close();
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::flush(uint64_t checkpoint) {
|
||||
LOG_INFO("Flush Vamana index, curDocs=%u", doc_cnt());
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
flush_header();
|
||||
return broker_->flush(checkpoint);
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::dump(const IndexDumper::Pointer &dumper) {
|
||||
LOG_INFO("Dump Vamana index, curDocs=%u", doc_cnt());
|
||||
|
||||
std::vector<key_t> keys(doc_cnt());
|
||||
auto ret = dump_segments(dumper, keys.data());
|
||||
if (ailego_unlikely(ret < 0)) {
|
||||
return static_cast<int>(ret);
|
||||
}
|
||||
*stats_.mutable_dumped_size() += ret;
|
||||
return 0;
|
||||
}
|
||||
|
||||
const VamanaEntity::Pointer VamanaStreamerEntity::clone() const {
|
||||
std::vector<Chunk::Pointer> cloned_chunks;
|
||||
cloned_chunks.reserve(node_chunks_.size());
|
||||
for (size_t i = 0; i < node_chunks_.size(); ++i) {
|
||||
cloned_chunks.emplace_back(node_chunks_[i]->clone());
|
||||
if (ailego_unlikely(!cloned_chunks[i])) {
|
||||
LOG_ERROR("VamanaStreamerEntity get chunk failed in clone");
|
||||
return VamanaEntity::Pointer();
|
||||
}
|
||||
}
|
||||
|
||||
auto *entity = new (std::nothrow) VamanaStreamerEntity(
|
||||
stats_, header(), chunk_size_, node_index_mask_bits_, get_vector_enabled_,
|
||||
use_key_info_map_, keys_map_lock_, keys_map_, std::move(cloned_chunks),
|
||||
broker_);
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("VamanaStreamerEntity new failed");
|
||||
}
|
||||
return VamanaEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
const VamanaEntity::Pointer VamanaMmapStreamerEntity::clone() const {
|
||||
std::vector<Chunk::Pointer> cloned_chunks;
|
||||
cloned_chunks.reserve(node_chunks_.size());
|
||||
for (size_t i = 0; i < node_chunks_.size(); ++i) {
|
||||
cloned_chunks.emplace_back(node_chunks_[i]->clone());
|
||||
if (ailego_unlikely(!cloned_chunks[i])) {
|
||||
LOG_ERROR("VamanaMmapStreamerEntity get chunk failed in clone");
|
||||
return VamanaEntity::Pointer();
|
||||
}
|
||||
}
|
||||
|
||||
auto *entity = new (std::nothrow) VamanaMmapStreamerEntity(
|
||||
stats_, header(), chunk_size_, node_index_mask_bits_, get_vector_enabled_,
|
||||
use_key_info_map_, keys_map_lock_, keys_map_, std::move(cloned_chunks),
|
||||
broker_);
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("VamanaMmapStreamerEntity new failed");
|
||||
}
|
||||
return VamanaEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
const VamanaEntity::Pointer VamanaContiguousStreamerEntity::clone() const {
|
||||
std::vector<Chunk::Pointer> cloned_chunks;
|
||||
cloned_chunks.reserve(node_chunks_.size());
|
||||
for (size_t i = 0; i < node_chunks_.size(); ++i) {
|
||||
cloned_chunks.emplace_back(node_chunks_[i]->clone());
|
||||
if (ailego_unlikely(!cloned_chunks[i])) {
|
||||
LOG_ERROR("VamanaContiguousStreamerEntity get chunk failed in clone");
|
||||
return VamanaEntity::Pointer();
|
||||
}
|
||||
}
|
||||
|
||||
auto *entity = new (std::nothrow) VamanaContiguousStreamerEntity(
|
||||
stats_, header(), chunk_size_, node_index_mask_bits_, get_vector_enabled_,
|
||||
use_key_info_map_, keys_map_lock_, keys_map_, std::move(cloned_chunks),
|
||||
broker_);
|
||||
if (ailego_unlikely(!entity)) {
|
||||
LOG_ERROR("VamanaContiguousStreamerEntity new failed");
|
||||
return VamanaEntity::Pointer();
|
||||
}
|
||||
|
||||
// Share contiguous memory with the clone (zero-copy)
|
||||
entity->node_memory_ = node_memory_;
|
||||
entity->node_base_ = node_base_;
|
||||
|
||||
return VamanaEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// VamanaContiguousStreamerEntity implementation
|
||||
// ============================================================================
|
||||
|
||||
char *VamanaContiguousStreamerEntity::allocate_contiguous(size_t size) {
|
||||
if (size == 0) return nullptr;
|
||||
#if defined(__linux__)
|
||||
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (ptr == MAP_FAILED) {
|
||||
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
|
||||
return nullptr;
|
||||
}
|
||||
::madvise(ptr, size, MADV_HUGEPAGE);
|
||||
return static_cast<char *>(ptr);
|
||||
#elif defined(__APPLE__)
|
||||
void *ptr = ::mmap(nullptr, size, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANON, -1, 0);
|
||||
if (ptr == MAP_FAILED) {
|
||||
LOG_ERROR("mmap failed for contiguous memory, size=%zu", size);
|
||||
return nullptr;
|
||||
}
|
||||
return static_cast<char *>(ptr);
|
||||
#elif defined(_WIN32)
|
||||
void *ptr = ::_aligned_malloc(size, ailego::MemoryHelper::PageSize());
|
||||
if (!ptr) {
|
||||
LOG_ERROR("_aligned_malloc failed for contiguous memory, size=%zu", size);
|
||||
return nullptr;
|
||||
}
|
||||
return static_cast<char *>(ptr);
|
||||
#else
|
||||
void *ptr = std::aligned_alloc(ailego::MemoryHelper::PageSize(), size);
|
||||
if (!ptr) {
|
||||
LOG_ERROR("aligned_alloc failed, size=%zu", size);
|
||||
return nullptr;
|
||||
}
|
||||
return static_cast<char *>(ptr);
|
||||
#endif
|
||||
}
|
||||
|
||||
int VamanaContiguousStreamerEntity::build_contiguous_memory() {
|
||||
node_memory_.reset();
|
||||
node_base_ = nullptr;
|
||||
|
||||
const uint32_t total_docs = doc_cnt();
|
||||
if (total_docs == 0) return 0;
|
||||
|
||||
const size_t per_node = node_size();
|
||||
const size_t total_node_data = static_cast<size_t>(total_docs) * per_node;
|
||||
size_t node_memory_size = AlignHugePageSize(total_node_data);
|
||||
char *raw_node = allocate_contiguous(node_memory_size);
|
||||
if (!raw_node) return IndexError_Runtime;
|
||||
node_memory_.reset(raw_node, ContiguousDeleter{node_memory_size});
|
||||
node_base_ = raw_node;
|
||||
|
||||
// Copy node data from chunks into contiguous memory
|
||||
const auto &chunks = node_chunks_;
|
||||
const uint32_t nodes_per_chunk = 1U << node_index_mask_bits_;
|
||||
for (size_t chunk_idx = 0; chunk_idx < chunks.size(); ++chunk_idx) {
|
||||
const void *chunk_data = nullptr;
|
||||
size_t data_size = chunks[chunk_idx]->data_size();
|
||||
chunks[chunk_idx]->read(0, &chunk_data, data_size);
|
||||
|
||||
uint32_t base_id = chunk_idx * nodes_per_chunk;
|
||||
uint32_t count_in_chunk = std::min(nodes_per_chunk, total_docs - base_id);
|
||||
|
||||
const char *src = static_cast<const char *>(chunk_data);
|
||||
char *dst = node_base_ + static_cast<size_t>(base_id) * per_node;
|
||||
std::memcpy(dst, src, static_cast<size_t>(count_in_chunk) * per_node);
|
||||
}
|
||||
|
||||
LOG_INFO(
|
||||
"Built Vamana contiguous memory: node_size=%zu total_docs=%u "
|
||||
"node_chunks=%zu",
|
||||
node_memory_size, total_docs, chunks.size());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Neighbor distance storage implementation (CSR-like, lazy-loaded)
|
||||
// ============================================================================
|
||||
|
||||
int VamanaStreamerEntity::ensure_dist_storage() {
|
||||
if (dist_loaded_) return 0;
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (dist_loaded_) return 0; // double-check after lock
|
||||
|
||||
dist_entry_size_ = static_cast<uint32_t>(max_degree() * sizeof(dist_t));
|
||||
|
||||
// Calculate how many dist chunks we need for existing nodes
|
||||
uint32_t total_docs = doc_cnt();
|
||||
if (total_docs == 0) {
|
||||
dist_loaded_ = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Check if dist chunks already exist in storage (reopened index)
|
||||
size_t existing_dist_chunks =
|
||||
broker_->get_chunk_cnt(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST);
|
||||
if (existing_dist_chunks > 0) {
|
||||
// Load existing dist chunks
|
||||
for (size_t i = 0; i < existing_dist_chunks; ++i) {
|
||||
auto chunk = broker_->get_chunk(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST, i);
|
||||
if (ailego_unlikely(!chunk)) {
|
||||
LOG_ERROR("Failed to load dist chunk %zu", i);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
dist_chunks_.emplace_back(std::move(chunk));
|
||||
}
|
||||
LOG_INFO("Loaded %zu existing dist chunks", existing_dist_chunks);
|
||||
} else {
|
||||
// Allocate new dist chunks for all existing nodes
|
||||
int ret = alloc_dist_chunks_for_existing_nodes();
|
||||
if (ret != 0) return ret;
|
||||
}
|
||||
|
||||
dist_loaded_ = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::ensure_dist_chunk_for(uint32_t chunk_index) {
|
||||
// No-op when dist storage is not active.
|
||||
if (!dist_loaded_ || dist_entry_size_ == 0) return 0;
|
||||
|
||||
// Idempotent: nothing to do if this dist chunk slot already exists and is
|
||||
// populated. (Slots created by the placeholder loop below will hold
|
||||
// nullptr and must still be (re-)allocated.)
|
||||
if (chunk_index < dist_chunks_.size() && dist_chunks_[chunk_index]) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t dist_chunk_data_size = node_cnt_per_chunk_ * dist_entry_size_;
|
||||
uint32_t dist_chunk_size = AlignPageSize(dist_chunk_data_size);
|
||||
auto dp = broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST,
|
||||
chunk_index, dist_chunk_size);
|
||||
if (ailego_unlikely(dp.first != 0)) {
|
||||
LOG_ERROR("Alloc dist chunk %u failed", chunk_index);
|
||||
return dp.first;
|
||||
}
|
||||
dp.second->resize(dist_chunk_data_size);
|
||||
while (dist_chunks_.size() <= chunk_index) {
|
||||
dist_chunks_.emplace_back(nullptr);
|
||||
}
|
||||
dist_chunks_[chunk_index] = std::move(dp.second);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int VamanaStreamerEntity::alloc_dist_chunks_for_existing_nodes() {
|
||||
uint32_t total_docs = doc_cnt();
|
||||
if (total_docs == 0) return 0;
|
||||
|
||||
// Calculate dist chunk size: same number of nodes per chunk as node chunks
|
||||
uint32_t dist_chunk_data_size = node_cnt_per_chunk_ * dist_entry_size_;
|
||||
uint32_t dist_chunk_size = AlignPageSize(dist_chunk_data_size);
|
||||
|
||||
uint32_t num_chunks_needed =
|
||||
(total_docs + node_cnt_per_chunk_ - 1) >> node_index_mask_bits_;
|
||||
|
||||
for (uint32_t i = 0; i < num_chunks_needed; ++i) {
|
||||
auto p = broker_->alloc_chunk(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST, i,
|
||||
dist_chunk_size);
|
||||
if (ailego_unlikely(p.first != 0)) {
|
||||
LOG_ERROR("Alloc dist chunk %u failed", i);
|
||||
return p.first;
|
||||
}
|
||||
// Resize to cover all nodes in this chunk
|
||||
uint32_t nodes_in_chunk =
|
||||
std::min(node_cnt_per_chunk_, total_docs - i * node_cnt_per_chunk_);
|
||||
size_t data_size = static_cast<size_t>(nodes_in_chunk) * dist_entry_size_;
|
||||
p.second->resize(data_size);
|
||||
dist_chunks_.emplace_back(std::move(p.second));
|
||||
}
|
||||
broker_->mark_dirty();
|
||||
LOG_INFO("Allocated %u dist chunks for %u existing nodes", num_chunks_needed,
|
||||
total_docs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const dist_t *VamanaStreamerEntity::get_neighbor_dists(node_id_t id) const {
|
||||
if (!dist_loaded_) return nullptr;
|
||||
|
||||
auto loc = get_dist_chunk_loc(id);
|
||||
if (ailego_unlikely(loc.first >= dist_chunks_.size())) {
|
||||
sync_dist_chunks(loc.first);
|
||||
}
|
||||
if (ailego_unlikely(loc.first >= dist_chunks_.size())) return nullptr;
|
||||
|
||||
const void *ptr = nullptr;
|
||||
size_t ret =
|
||||
dist_chunks_[loc.first]->read(loc.second, &ptr, dist_entry_size_);
|
||||
if (ailego_unlikely(ret != dist_entry_size_)) return nullptr;
|
||||
return static_cast<const dist_t *>(ptr);
|
||||
}
|
||||
|
||||
void VamanaStreamerEntity::update_neighbor_dists(
|
||||
node_id_t id, const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
|
||||
if (!dist_loaded_) return;
|
||||
|
||||
auto loc = get_dist_chunk_loc(id);
|
||||
// Dist chunk must have been pre-allocated by add_vector or
|
||||
// ensure_dist_storage
|
||||
if (ailego_unlikely(loc.first >= dist_chunks_.size() ||
|
||||
dist_chunks_[loc.first] == nullptr)) {
|
||||
LOG_ERROR("Dist chunk %u not allocated for node %u", loc.first, id);
|
||||
return;
|
||||
}
|
||||
|
||||
// Write distances: fill max_degree slots, zero-pad unused slots
|
||||
uint32_t max_deg = static_cast<uint32_t>(max_degree());
|
||||
std::vector<dist_t> dists(max_deg, 0.0f);
|
||||
for (size_t i = 0; i < neighbors.size() && i < max_deg; ++i) {
|
||||
dists[i] = neighbors[i].second;
|
||||
}
|
||||
dist_chunks_[loc.first]->write(loc.second, dists.data(), dist_entry_size_);
|
||||
}
|
||||
|
||||
void VamanaStreamerEntity::set_neighbor_dist(node_id_t id, uint32_t idx,
|
||||
dist_t dist) {
|
||||
if (!dist_loaded_) return;
|
||||
|
||||
auto loc = get_dist_chunk_loc(id);
|
||||
if (ailego_unlikely(loc.first >= dist_chunks_.size())) return;
|
||||
|
||||
uint32_t offset = loc.second + idx * sizeof(dist_t);
|
||||
dist_chunks_[loc.first]->write(offset, &dist, sizeof(dist_t));
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,635 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#if defined(__linux__) || defined(__APPLE__)
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <sparsehash/dense_hash_map>
|
||||
#include <zvec/ailego/container/heap.h>
|
||||
#include <zvec/core/framework/index_framework.h>
|
||||
#include "algorithm/hnsw/hnsw_chunk.h"
|
||||
#include "algorithm/hnsw/hnsw_entity.h" // MmapMemoryBlock, BufferPoolMemoryBlock, NeighborsT
|
||||
#include "vamana_entity.h"
|
||||
#include "vamana_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
|
||||
// Storage mode for VamanaStreamerEntity
|
||||
enum class VamanaStorageMode { kMmap = 0, kBufferPool = 1, kContiguous = 2 };
|
||||
|
||||
// VamanaStreamerEntity manages vector data, primary keys, and neighbors
|
||||
// for a single-layer Vamana graph in streaming (incremental) mode.
|
||||
// Unlike HNSW, Vamana has no upper-level neighbors — only a single
|
||||
// neighbor list per node. Node layout in chunk:
|
||||
// [vector_data (vector_size) | key (sizeof(key_t)) | NeighborsHeader +
|
||||
// neighbors (neighbors_size)]
|
||||
class VamanaStreamerEntity : public VamanaEntity {
|
||||
public:
|
||||
// Virtual interface implementation
|
||||
virtual int cleanup() override;
|
||||
virtual const VamanaEntity::Pointer clone() const override;
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
virtual const void *get_vector(node_id_t id) const override;
|
||||
virtual int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
virtual int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
virtual int get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
|
||||
virtual const Neighbors get_neighbors(node_id_t id) const override;
|
||||
|
||||
virtual int add_vector(key_t key, const void *vec, node_id_t *id) override;
|
||||
virtual int add_vector_with_id(node_id_t id, const void *vec) override;
|
||||
virtual int update_neighbors(
|
||||
node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
|
||||
virtual void add_neighbor(node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
virtual void update_entry_point(node_id_t ep) override;
|
||||
|
||||
// --- Neighbor distance storage ---
|
||||
int ensure_dist_storage() override;
|
||||
bool dist_storage_loaded() const override {
|
||||
return dist_loaded_;
|
||||
}
|
||||
const dist_t *get_neighbor_dists(node_id_t id) const override;
|
||||
void update_neighbor_dists(
|
||||
node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
|
||||
void set_neighbor_dist(node_id_t id, uint32_t idx, dist_t dist) override;
|
||||
|
||||
virtual VamanaStorageMode storage_mode() const {
|
||||
return VamanaStorageMode::kMmap;
|
||||
}
|
||||
|
||||
void set_use_key_info_map(bool use_id_map) {
|
||||
use_key_info_map_ = use_id_map;
|
||||
}
|
||||
|
||||
public:
|
||||
VamanaStreamerEntity(IndexStreamer::Stats &stats);
|
||||
~VamanaStreamerEntity();
|
||||
|
||||
virtual const void *get_vector_by_key(key_t key) const override {
|
||||
auto id = get_id(key);
|
||||
return id == kInvalidNodeId ? nullptr : get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_key(
|
||||
const key_t key, IndexStorage::MemoryBlock &block) const override {
|
||||
auto id = get_id(key);
|
||||
if (id != kInvalidNodeId) {
|
||||
return get_vector(id, block);
|
||||
}
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
int init(size_t max_doc_cnt);
|
||||
int flush(uint64_t checkpoint);
|
||||
int open(IndexStorage::Pointer stg, uint64_t max_index_size, bool check_crc);
|
||||
int close();
|
||||
|
||||
int set_index_meta(const IndexMeta &meta) const {
|
||||
return IndexHelper::SerializeToStorage(meta, broker_->storage().get());
|
||||
}
|
||||
|
||||
int get_index_meta(IndexMeta *meta) const {
|
||||
return IndexHelper::DeserializeFromStorage(broker_->storage().get(), meta);
|
||||
}
|
||||
|
||||
inline void set_chunk_size(size_t val) {
|
||||
chunk_size_ = val;
|
||||
}
|
||||
inline void set_get_vector(bool val) {
|
||||
get_vector_enabled_ = val;
|
||||
}
|
||||
|
||||
inline node_id_t get_id(key_t key) const {
|
||||
if (use_key_info_map_) {
|
||||
keys_map_lock_->lock_shared();
|
||||
auto it = keys_map_->find(key);
|
||||
keys_map_lock_->unlock_shared();
|
||||
return it == keys_map_->end() ? kInvalidNodeId : it->second;
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
// --- Typed access methods for hot-path optimization ---
|
||||
// These are templated on MemBlock type to avoid runtime branching.
|
||||
|
||||
template <typename MemBlock>
|
||||
inline NeighborsT<MemBlock> get_neighbors_typed(node_id_t id) const;
|
||||
|
||||
template <typename MemBlock>
|
||||
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
|
||||
std::vector<MemBlock> &vec_blocks) const;
|
||||
|
||||
template <typename MemBlock>
|
||||
inline key_t get_key_typed(node_id_t id) const;
|
||||
|
||||
protected:
|
||||
inline void sync_node_chunks(size_t idx) const {
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, idx, &node_chunks_);
|
||||
}
|
||||
|
||||
protected:
|
||||
template <class Key, class T>
|
||||
using HashMap = google::dense_hash_map<Key, T, std::hash<Key>>;
|
||||
template <class Key, class T>
|
||||
using HashMapPointer = std::shared_ptr<HashMap<Key, T>>;
|
||||
|
||||
//! Clone constructor, used by clone method in subclasses
|
||||
VamanaStreamerEntity(IndexStreamer::Stats &stats, const VamanaHeader &hd,
|
||||
size_t chunk_size, uint32_t node_index_mask_bits,
|
||||
bool get_vector_enabled, bool use_key_info_map,
|
||||
std::shared_ptr<ailego::SharedMutex> &keys_map_lock,
|
||||
const HashMapPointer<key_t, node_id_t> &keys_map,
|
||||
std::vector<Chunk::Pointer> &&node_chunks,
|
||||
const ChunkBroker::Pointer &broker)
|
||||
: stats_(stats),
|
||||
chunk_size_(chunk_size),
|
||||
node_index_mask_bits_(node_index_mask_bits),
|
||||
node_cnt_per_chunk_(1UL << node_index_mask_bits_),
|
||||
node_index_mask_(node_cnt_per_chunk_ - 1),
|
||||
get_vector_enabled_(get_vector_enabled),
|
||||
use_key_info_map_(use_key_info_map),
|
||||
keys_map_lock_(keys_map_lock),
|
||||
keys_map_(keys_map),
|
||||
node_chunks_(std::move(node_chunks)),
|
||||
broker_(broker) {
|
||||
*mutable_header() = hd;
|
||||
neighbor_size_ = neighbors_size();
|
||||
}
|
||||
|
||||
//! Lazy chunk synchronization: fetches chunks from broker when needed.
|
||||
//! Each clone entity has its own node_chunks_ vector, so concurrent
|
||||
//! search threads do not race with the writer's emplace_back.
|
||||
void sync_chunks(ChunkBroker::CHUNK_TYPE type, size_t idx,
|
||||
std::vector<Chunk::Pointer> *chunks) const {
|
||||
if (ailego_likely(idx < chunks->size())) {
|
||||
return;
|
||||
}
|
||||
for (size_t i = chunks->size(); i <= idx; ++i) {
|
||||
auto chunk = broker_->get_chunk(type, i);
|
||||
ailego_assert_with(!!chunk, "get chunk failed");
|
||||
chunks->emplace_back(std::move(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
inline std::pair<uint32_t, uint32_t> get_vector_chunk_loc(
|
||||
node_id_t id) const {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset = (id & node_index_mask_) * node_size();
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
|
||||
return std::make_pair(chunk_idx, offset);
|
||||
}
|
||||
|
||||
inline std::pair<uint32_t, uint32_t> get_key_chunk_loc(node_id_t id) const {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
|
||||
return std::make_pair(chunk_idx, offset);
|
||||
}
|
||||
|
||||
inline std::pair<Chunk *, size_t> get_neighbor_chunk_loc(node_id_t id) const {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset =
|
||||
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
|
||||
ailego_assert_abort(chunk_idx < node_chunks_.size(), "invalid chunk idx");
|
||||
return std::make_pair(node_chunks_[chunk_idx].get(), offset);
|
||||
}
|
||||
|
||||
// Get chunk location for neighbor distance data.
|
||||
// Uses the same chunk indexing as node chunks but with dist_entry_size_.
|
||||
inline std::pair<uint32_t, uint32_t> get_dist_chunk_loc(node_id_t id) const {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset = (id & node_index_mask_) * dist_entry_size_;
|
||||
return std::make_pair(chunk_idx, offset);
|
||||
}
|
||||
|
||||
void sync_dist_chunks(size_t idx) const {
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NEIGHBOR_DIST, idx, &dist_chunks_);
|
||||
}
|
||||
|
||||
int ensure_dist_chunk_for(uint32_t chunk_index);
|
||||
|
||||
int alloc_dist_chunks_for_existing_nodes();
|
||||
|
||||
size_t estimate_doc_capacity() const {
|
||||
return node_chunks_.capacity() * node_cnt_per_chunk_;
|
||||
}
|
||||
|
||||
int init_chunk_params(size_t max_index_size, bool huge_page) {
|
||||
node_cnt_per_chunk_ = std::max<uint32_t>(1, chunk_size_ / node_size());
|
||||
node_index_mask_bits_ = std::ceil(std::log2(node_cnt_per_chunk_));
|
||||
node_cnt_per_chunk_ = 1UL << node_index_mask_bits_;
|
||||
if (huge_page) {
|
||||
chunk_size_ = AlignHugePageSize(node_cnt_per_chunk_ * node_size());
|
||||
} else {
|
||||
chunk_size_ = AlignPageSize(node_cnt_per_chunk_ * node_size());
|
||||
}
|
||||
node_index_mask_ = node_cnt_per_chunk_ - 1;
|
||||
|
||||
if (max_index_size == 0UL) {
|
||||
max_index_size_ = chunk_size_ * kDefaultMaxChunkCnt;
|
||||
} else {
|
||||
max_index_size_ = max_index_size;
|
||||
}
|
||||
|
||||
size_t max_node_chunk_cnt =
|
||||
std::ceil(static_cast<double>(max_index_size_) / chunk_size_);
|
||||
node_chunks_.reserve(max_node_chunk_cnt);
|
||||
|
||||
LOG_DEBUG(
|
||||
"VamanaSettings: nodeSize=%zu chunkSize=%u nodeCntPerChunk=%u "
|
||||
"maxChunkCnt=%zu maxIndexSize=%zu",
|
||||
node_size(), chunk_size_, node_cnt_per_chunk_, max_node_chunk_cnt,
|
||||
max_index_size_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int init_chunks(const Chunk::Pointer &header_chunk);
|
||||
|
||||
int flush_header(void) {
|
||||
if (!broker_->dirty()) {
|
||||
return 0;
|
||||
}
|
||||
auto header_chunk = broker_->get_chunk(ChunkBroker::CHUNK_TYPE_HEADER,
|
||||
ChunkBroker::kDefaultChunkSeqId);
|
||||
if (ailego_unlikely(!header_chunk)) {
|
||||
LOG_ERROR("get header chunk failed");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
size_t size = header_chunk->write(0UL, &header(), header_size());
|
||||
if (ailego_unlikely(size != header_size())) {
|
||||
LOG_ERROR("Write header chunk failed");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
protected:
|
||||
IndexStreamer::Stats &stats_;
|
||||
std::mutex mutex_{};
|
||||
size_t max_index_size_{0UL};
|
||||
uint32_t chunk_size_{kDefaultChunkSize};
|
||||
uint32_t node_index_mask_bits_{0U};
|
||||
uint32_t node_cnt_per_chunk_{0U};
|
||||
uint32_t node_index_mask_{0U};
|
||||
uint32_t neighbor_size_{0U};
|
||||
bool get_vector_enabled_{false};
|
||||
bool use_key_info_map_{true};
|
||||
|
||||
mutable std::shared_ptr<ailego::SharedMutex> keys_map_lock_;
|
||||
HashMapPointer<key_t, node_id_t> keys_map_;
|
||||
|
||||
ChunkBroker::Pointer broker_;
|
||||
|
||||
mutable std::vector<Chunk::Pointer> node_chunks_{};
|
||||
|
||||
private:
|
||||
mutable std::vector<Chunk::Pointer> dist_chunks_{};
|
||||
bool dist_loaded_{false};
|
||||
uint32_t dist_entry_size_{0}; // max_degree * sizeof(dist_t)
|
||||
};
|
||||
|
||||
// --- Template specializations for typed MemoryBlock access ---
|
||||
|
||||
template <>
|
||||
inline NeighborsT<MmapMemoryBlock>
|
||||
VamanaStreamerEntity::get_neighbors_typed<MmapMemoryBlock>(node_id_t id) const {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset =
|
||||
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
|
||||
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
|
||||
ailego_assert_with(offset < node_chunks_[chunk_idx]->data_size(),
|
||||
"invalid chunk offset");
|
||||
const void *ptr = nullptr;
|
||||
size_t ret = node_chunks_[chunk_idx]->read(offset, &ptr, neighbor_size_);
|
||||
if (ailego_unlikely(ret != neighbor_size_)) {
|
||||
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
|
||||
return NeighborsT<MmapMemoryBlock>();
|
||||
}
|
||||
MmapMemoryBlock block(const_cast<void *>(ptr));
|
||||
return NeighborsT<MmapMemoryBlock>(std::move(block));
|
||||
}
|
||||
|
||||
template <>
|
||||
inline NeighborsT<BufferPoolMemoryBlock>
|
||||
VamanaStreamerEntity::get_neighbors_typed<BufferPoolMemoryBlock>(
|
||||
node_id_t id) const {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset =
|
||||
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
|
||||
sync_chunks(ChunkBroker::CHUNK_TYPE_NODE, chunk_idx, &node_chunks_);
|
||||
ailego_assert_with(chunk_idx < node_chunks_.size(), "invalid chunk idx");
|
||||
IndexStorage::MemoryBlock mem_block;
|
||||
size_t ret = node_chunks_[chunk_idx]->read(offset, mem_block, neighbor_size_);
|
||||
if (ailego_unlikely(ret != neighbor_size_)) {
|
||||
LOG_ERROR("Read neighbor header failed, ret=%zu", ret);
|
||||
return NeighborsT<BufferPoolMemoryBlock>();
|
||||
}
|
||||
BufferPoolMemoryBlock block(mem_block.buffer_pool_handle_,
|
||||
mem_block.buffer_block_id_, mem_block.data_);
|
||||
mem_block.buffer_pool_handle_ = nullptr;
|
||||
return NeighborsT<BufferPoolMemoryBlock>(std::move(block));
|
||||
}
|
||||
|
||||
template <>
|
||||
inline int VamanaStreamerEntity::get_vector_typed<MmapMemoryBlock>(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<MmapMemoryBlock> &vec_blocks) const {
|
||||
vec_blocks.resize(count);
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
auto loc = get_vector_chunk_loc(ids[i]);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
const void *ptr = nullptr;
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, vector_size());
|
||||
if (ailego_unlikely(ret != vector_size())) {
|
||||
LOG_ERROR("Read vector failed, ret=%zu", ret);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
vec_blocks[i].reset(const_cast<void *>(ptr));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline int VamanaStreamerEntity::get_vector_typed<BufferPoolMemoryBlock>(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<BufferPoolMemoryBlock> &vec_blocks) const {
|
||||
vec_blocks.resize(count);
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
auto loc = get_vector_chunk_loc(ids[i]);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
IndexStorage::MemoryBlock mem_block;
|
||||
size_t ret =
|
||||
node_chunks_[loc.first]->read(loc.second, mem_block, vector_size());
|
||||
if (ailego_unlikely(ret != vector_size())) {
|
||||
LOG_ERROR("Read vector failed, ret=%zu", ret);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
vec_blocks[i] =
|
||||
BufferPoolMemoryBlock(mem_block.buffer_pool_handle_,
|
||||
mem_block.buffer_block_id_, mem_block.data_);
|
||||
mem_block.buffer_pool_handle_ = nullptr;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline key_t VamanaStreamerEntity::get_key_typed<MmapMemoryBlock>(
|
||||
node_id_t id) const {
|
||||
if (!use_key_info_map_) return id;
|
||||
auto loc = get_key_chunk_loc(id);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
const void *ptr = nullptr;
|
||||
size_t ret = node_chunks_[loc.first]->read(loc.second, &ptr, sizeof(key_t));
|
||||
if (ailego_unlikely(ret != sizeof(key_t))) {
|
||||
LOG_ERROR("Read key failed, ret=%zu", ret);
|
||||
return kInvalidKey;
|
||||
}
|
||||
return *reinterpret_cast<const key_t *>(ptr);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline key_t VamanaStreamerEntity::get_key_typed<BufferPoolMemoryBlock>(
|
||||
node_id_t id) const {
|
||||
if (!use_key_info_map_) return id;
|
||||
auto loc = get_key_chunk_loc(id);
|
||||
ailego_assert_with(loc.first < node_chunks_.size(), "invalid chunk idx");
|
||||
IndexStorage::MemoryBlock key_block;
|
||||
size_t ret =
|
||||
node_chunks_[loc.first]->read(loc.second, key_block, sizeof(key_t));
|
||||
if (ailego_unlikely(ret != sizeof(key_t))) {
|
||||
LOG_ERROR("Read key failed, ret=%zu", ret);
|
||||
return kInvalidKey;
|
||||
}
|
||||
return *reinterpret_cast<const key_t *>(key_block.data());
|
||||
}
|
||||
|
||||
// --- Typed entity subclass for mmap mode ---
|
||||
// Caches chunk base addresses to eliminate virtual function calls on the
|
||||
// search hot path. For mmap mode, chunk data is memory-mapped at init time,
|
||||
// so we can directly compute pointers via base_addr + offset.
|
||||
class VamanaMmapStreamerEntity : public VamanaStreamerEntity {
|
||||
public:
|
||||
using MemoryBlock = MmapMemoryBlock;
|
||||
using TypedNeighbors = NeighborsT<MmapMemoryBlock>;
|
||||
|
||||
using VamanaStreamerEntity::VamanaStreamerEntity;
|
||||
|
||||
VamanaStorageMode storage_mode() const override {
|
||||
return VamanaStorageMode::kMmap;
|
||||
}
|
||||
|
||||
//! Override clone to return correct subclass type, so that
|
||||
//! static_cast<const VamanaMmapStreamerEntity&> in the algorithm is safe.
|
||||
const VamanaEntity::Pointer clone() const override;
|
||||
|
||||
inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset =
|
||||
(id & node_index_mask_) * node_size() + vector_size() + sizeof(key_t);
|
||||
const char *base = get_node_chunk_base(chunk_idx);
|
||||
MmapMemoryBlock block(const_cast<char *>(base + offset));
|
||||
return TypedNeighbors(std::move(block));
|
||||
}
|
||||
|
||||
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
|
||||
std::vector<MmapMemoryBlock> &vec_blocks) const {
|
||||
vec_blocks.resize(count);
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
uint32_t chunk_idx = ids[i] >> node_index_mask_bits_;
|
||||
uint32_t offset = (ids[i] & node_index_mask_) * node_size();
|
||||
const char *base = get_node_chunk_base(chunk_idx);
|
||||
vec_blocks[i].reset(const_cast<char *>(base + offset));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline key_t get_key_typed(node_id_t id) const {
|
||||
if (!use_key_info_map_) return id;
|
||||
uint32_t chunk_idx = id >> node_index_mask_bits_;
|
||||
uint32_t offset = (id & node_index_mask_) * node_size() + vector_size();
|
||||
const char *base = get_node_chunk_base(chunk_idx);
|
||||
return *reinterpret_cast<const key_t *>(base + offset);
|
||||
}
|
||||
|
||||
private:
|
||||
inline const char *get_node_chunk_base(uint32_t chunk_idx) const {
|
||||
if (ailego_unlikely(chunk_idx >= node_chunk_bases_.size())) {
|
||||
sync_node_chunk_bases(chunk_idx);
|
||||
}
|
||||
return node_chunk_bases_[chunk_idx];
|
||||
}
|
||||
|
||||
void sync_node_chunk_bases(uint32_t chunk_idx) const {
|
||||
sync_node_chunks(chunk_idx);
|
||||
const auto &chunks = node_chunks_;
|
||||
for (size_t i = node_chunk_bases_.size(); i <= chunk_idx; ++i) {
|
||||
const void *ptr = nullptr;
|
||||
chunks[i]->read(0, &ptr, 1);
|
||||
node_chunk_bases_.push_back(static_cast<const char *>(ptr));
|
||||
}
|
||||
}
|
||||
|
||||
mutable std::vector<const char *> node_chunk_bases_{};
|
||||
};
|
||||
|
||||
// --- Typed entity subclass for buffer pool mode ---
|
||||
class VamanaBufferPoolStreamerEntity : public VamanaStreamerEntity {
|
||||
public:
|
||||
using MemoryBlock = BufferPoolMemoryBlock;
|
||||
using TypedNeighbors = NeighborsT<BufferPoolMemoryBlock>;
|
||||
|
||||
using VamanaStreamerEntity::VamanaStreamerEntity;
|
||||
|
||||
VamanaStorageMode storage_mode() const override {
|
||||
return VamanaStorageMode::kBufferPool;
|
||||
}
|
||||
|
||||
inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
|
||||
return VamanaStreamerEntity::get_neighbors_typed<BufferPoolMemoryBlock>(id);
|
||||
}
|
||||
|
||||
inline int get_vector_typed(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<BufferPoolMemoryBlock> &vec_blocks) const {
|
||||
return VamanaStreamerEntity::get_vector_typed<BufferPoolMemoryBlock>(
|
||||
ids, count, vec_blocks);
|
||||
}
|
||||
|
||||
inline key_t get_key_typed(node_id_t id) const {
|
||||
return VamanaStreamerEntity::get_key_typed<BufferPoolMemoryBlock>(id);
|
||||
}
|
||||
};
|
||||
|
||||
// --- Typed entity subclass for contiguous memory mode ---
|
||||
// Allocates contiguous memory and copies all chunk data into it.
|
||||
// Access is via a single base pointer + offset, eliminating chunk-level
|
||||
// indirection and maximizing memory locality.
|
||||
class VamanaContiguousStreamerEntity : public VamanaMmapStreamerEntity {
|
||||
public:
|
||||
using VamanaMmapStreamerEntity::VamanaMmapStreamerEntity;
|
||||
|
||||
VamanaStorageMode storage_mode() const override {
|
||||
return VamanaStorageMode::kContiguous;
|
||||
}
|
||||
|
||||
//! Override clone to return correct subclass type.
|
||||
//! Cloned entity shares contiguous memory via shared_ptr.
|
||||
const VamanaEntity::Pointer clone() const override;
|
||||
|
||||
~VamanaContiguousStreamerEntity() = default;
|
||||
|
||||
// Build contiguous memory from chunks after open.
|
||||
int build_contiguous_memory();
|
||||
|
||||
//! Degrade to mmap mode by releasing contiguous memory and falling back
|
||||
//! to chunk-based access.
|
||||
void degrade_to_mmap() {
|
||||
node_memory_.reset();
|
||||
node_base_ = nullptr;
|
||||
LOG_INFO("Vamana contiguous entity degraded to mmap mode for insertion");
|
||||
}
|
||||
|
||||
bool is_contiguous() const {
|
||||
return node_base_ != nullptr;
|
||||
}
|
||||
|
||||
int add_vector(key_t key, const void *vec, node_id_t *id) override {
|
||||
if (ailego_unlikely(is_contiguous())) degrade_to_mmap();
|
||||
return VamanaMmapStreamerEntity::add_vector(key, vec, id);
|
||||
}
|
||||
|
||||
int add_vector_with_id(node_id_t id, const void *vec) override {
|
||||
if (ailego_unlikely(is_contiguous())) degrade_to_mmap();
|
||||
return VamanaMmapStreamerEntity::add_vector_with_id(id, vec);
|
||||
}
|
||||
|
||||
inline TypedNeighbors get_neighbors_typed(node_id_t id) const {
|
||||
if (ailego_likely(node_base_ != nullptr)) {
|
||||
const char *ptr = node_base_ + static_cast<size_t>(id) * node_size() +
|
||||
vector_size() + sizeof(key_t);
|
||||
MmapMemoryBlock block(const_cast<char *>(ptr));
|
||||
return TypedNeighbors(std::move(block));
|
||||
}
|
||||
return VamanaMmapStreamerEntity::get_neighbors_typed(id);
|
||||
}
|
||||
|
||||
inline int get_vector_typed(const node_id_t *ids, uint32_t count,
|
||||
std::vector<MmapMemoryBlock> &vec_blocks) const {
|
||||
if (ailego_likely(node_base_ != nullptr)) {
|
||||
vec_blocks.resize(count);
|
||||
for (auto i = 0U; i < count; ++i) {
|
||||
const char *ptr =
|
||||
node_base_ + static_cast<size_t>(ids[i]) * node_size();
|
||||
vec_blocks[i].reset(const_cast<char *>(ptr));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
return VamanaMmapStreamerEntity::get_vector_typed(ids, count, vec_blocks);
|
||||
}
|
||||
|
||||
inline key_t get_key_typed(node_id_t id) const {
|
||||
if (ailego_likely(node_base_ != nullptr)) {
|
||||
if (!use_key_info_map_) return id;
|
||||
const char *ptr =
|
||||
node_base_ + static_cast<size_t>(id) * node_size() + vector_size();
|
||||
return *reinterpret_cast<const key_t *>(ptr);
|
||||
}
|
||||
return VamanaMmapStreamerEntity::get_key_typed(id);
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Custom deleter for contiguous memory (munmap / _aligned_free / free)
|
||||
struct ContiguousDeleter {
|
||||
size_t size;
|
||||
void operator()(char *ptr) const {
|
||||
if (!ptr) return;
|
||||
#if defined(__linux__) || defined(__APPLE__)
|
||||
::munmap(ptr, size);
|
||||
#elif defined(_WIN32)
|
||||
::_aligned_free(ptr);
|
||||
#else
|
||||
std::free(ptr);
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
//! Shared ownership of contiguous memory (enables zero-copy clone)
|
||||
std::shared_ptr<char> node_memory_{};
|
||||
|
||||
//! Raw pointer for hot-path access (derived from shared_ptr)
|
||||
char *node_base_{nullptr};
|
||||
|
||||
private:
|
||||
static char *allocate_contiguous(size_t size);
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
Reference in New Issue
Block a user