feat: buffer storage write (#414)

This commit is contained in:
ZeFeng Yin
2026-06-01 10:40:49 +08:00
committed by GitHub
parent de8fb760ef
commit 74beb2a828
24 changed files with 4030 additions and 731 deletions
+1 -1
View File
@@ -34,7 +34,7 @@ FlatStreamer<BATCH_SIZE>::FlatStreamer() : entity_(stats_) {}
template <size_t BATCH_SIZE>
FlatStreamer<BATCH_SIZE>::~FlatStreamer() {
if (state_ == STATE_INITED) {
if (state_ == STATE_INITED || state_ == STATE_OPENED) {
this->cleanup();
}
}
@@ -165,13 +165,20 @@ int FlatStreamerEntity::add(uint64_t key, const void *vec, size_t size) {
IndexStorage::MemoryBlock head_block;
this->get_head_block(head_block);
const BlockLocation *bl =
reinterpret_cast<const BlockLocation *>(head_block.data());
if (ailego_unlikely(bl == nullptr)) {
LOG_ERROR("Failed to get block loc");
return IndexError_ReadData;
BlockLocation block;
{
const BlockLocation *bl =
reinterpret_cast<const BlockLocation *>(head_block.data());
if (ailego_unlikely(bl == nullptr)) {
LOG_ERROR("Failed to get block loc");
return IndexError_ReadData;
}
block = *bl;
}
BlockLocation block = *bl;
// Release the head block reference early so that the buffer pool ref_count
// and memory budget held by it do not block subsequent acquire/evict in this
// function (alloc_block / add_to_block may compete for the same memory).
head_block.reset(nullptr);
if (!this->is_valid_block(block)) {
int ret = this->alloc_block(block, &block);
@@ -922,6 +929,9 @@ int FlatStreamerEntity::add_vector_with_id(const uint32_t id, const void *query,
this->get_head_block(head_block);
BlockLocation block =
*reinterpret_cast<const BlockLocation *>(head_block.data());
// Release buffer-pool pin before any alloc_block() call that may trigger
// append_segment() and rebuild the pool (same reason as in add()).
head_block.reset(nullptr);
if (!this->is_valid_block(block)) {
int ret = this->alloc_block(block, &block);
if (ailego_unlikely(ret != 0)) {
+17 -22
View File
@@ -41,9 +41,9 @@ class HnswIndexHashMap {
items_(reinterpret_cast<const Item *>(data)) {}
//! Return a empty loc or the key item loc
Slot(Chunk::Pointer &&chunk, IndexStorage::MemoryBlock &&mem_block)
: chunk_(std::move(chunk)), items_block_(std::move(mem_block)) {
items_ = reinterpret_cast<const Item *>(items_block_.data());
Slot(Chunk::Pointer &&chunk, std::vector<char> &&local_data)
: chunk_(std::move(chunk)), local_data_(std::move(local_data)) {
items_ = reinterpret_cast<const Item *>(local_data_.data());
}
const_iterator find(key_type key, uint32_t max_items, uint32_t mask) const {
auto it = &items_[key & mask];
@@ -73,8 +73,8 @@ class HnswIndexHashMap {
private:
Chunk::Pointer chunk_{};
const Item *items_{nullptr}; // point to chunk data
IndexStorage::MemoryBlock items_block_{};
const Item *items_{nullptr}; // point to local_data_
std::vector<char> local_data_{};
};
public:
@@ -114,9 +114,9 @@ class HnswIndexHashMap {
}
int cleanup(void) {
broker_.reset();
slots_.clear();
slots_.shrink_to_fit();
broker_.reset();
mask_bits_ = 0U;
slot_items_ = 0U;
slot_loc_mask_ = 0U;
@@ -141,7 +141,6 @@ class HnswIndexHashMap {
auto idx = key >> mask_bits_;
if (idx >= slots_.size()) {
if (ailego_unlikely(idx >= slots_.capacity())) {
LOG_ERROR("no space to insert");
return false;
}
for (auto i = slots_.size(); i <= idx; ++i) {
@@ -152,7 +151,6 @@ class HnswIndexHashMap {
}
auto it = slots_[idx].find(key, slot_items_, slot_loc_mask_);
if (ailego_unlikely(it == nullptr)) {
LOG_ERROR("no space to insert");
return false;
}
@@ -179,14 +177,10 @@ class HnswIndexHashMap {
LOG_ERROR("Chunk resize failed, size=%zu", size);
return false;
}
//! Read the whole data to memory
IndexStorage::MemoryBlock data_block;
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
LOG_ERROR("Chunk read failed, size=%zu", size);
return false;
}
slots_.emplace_back(std::move(chunk), std::move(data_block));
//! Use a local zero-initialized buffer; new chunks contain all zeros,
//! so no buffer-pool read is needed and no ref_count is pinned.
std::vector<char> local_buf(size, 0);
slots_.emplace_back(std::move(chunk), std::move(local_buf));
return true;
}
@@ -208,13 +202,14 @@ class HnswIndexHashMap {
i, chunk->data_size(), size);
return IndexError_InvalidFormat;
}
//! Read the whole data to memory
IndexStorage::MemoryBlock data_block;
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
LOG_ERROR("Chunk read failed, size=%zu", size);
return false;
//! Copy chunk data into a local buffer via fetch() so that no
//! buffer-pool block is pinned for the lifetime of the Slot.
std::vector<char> local_buf(size);
if (ailego_unlikely(chunk->fetch(0U, local_buf.data(), size) != size)) {
LOG_ERROR("Chunk fetch failed, size=%zu", size);
return IndexError_InvalidFormat;
}
slots_.emplace_back(std::move(chunk), std::move(data_block));
slots_.emplace_back(std::move(chunk), std::move(local_buf));
}
return 0;
}
+1 -1
View File
@@ -28,7 +28,7 @@ namespace core {
HnswStreamer::HnswStreamer() = default;
HnswStreamer::~HnswStreamer() {
if (state_ == STATE_INITED) {
if (state_ == STATE_INITED || state_ == STATE_OPENED) {
this->cleanup();
}
}
@@ -37,6 +37,7 @@ int HnswStreamerEntity::init(size_t max_doc_cnt) {
std::lock_guard<std::mutex> lock(mutex_);
broker_ = std::make_shared<ChunkBroker>(stats_);
upper_neighbor_index_ = std::make_shared<NIHashMap>();
upper_neighbor_rw_mutex_ = std::make_shared<std::shared_mutex>();
keys_map_lock_ = std::make_shared<ailego::SharedMutex>();
keys_map_ = std::make_shared<HashMap<key_t, node_id_t>>();
if (!keys_map_ || !upper_neighbor_index_ || !broker_ || !keys_map_lock_) {
@@ -767,9 +768,10 @@ const HnswEntity::Pointer HnswStreamerEntity::clone() const {
HnswStreamerEntity *entity = new (std::nothrow) HnswStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_,
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_,
node_chunk_bases_, upper_neighbor_chunk_bases_);
upper_neighbor_index_, upper_neighbor_rw_mutex_, keys_map_lock_,
keys_map_, use_key_info_map_, std::move(node_chunks),
std::move(upper_neighbor_chunks), broker_, node_chunk_bases_,
upper_neighbor_chunk_bases_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswStreamerEntity new failed");
}
@@ -800,9 +802,9 @@ const HnswEntity::Pointer HnswMmapStreamerEntity::clone() const {
auto *entity = new (std::nothrow) HnswMmapStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_,
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_,
nullptr, nullptr);
upper_neighbor_index_, upper_neighbor_rw_mutex_, keys_map_lock_,
keys_map_, use_key_info_map_, std::move(node_chunks),
std::move(upper_neighbor_chunks), broker_, nullptr, nullptr);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswMmapStreamerEntity new failed");
}
@@ -833,9 +835,9 @@ const HnswEntity::Pointer HnswContiguousStreamerEntity::clone() const {
auto *entity = new (std::nothrow) HnswContiguousStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_,
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_,
nullptr, nullptr);
upper_neighbor_index_, upper_neighbor_rw_mutex_, keys_map_lock_,
keys_map_, use_key_info_map_, std::move(node_chunks),
std::move(upper_neighbor_chunks), broker_, nullptr, nullptr);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswContiguousStreamerEntity new failed");
return HnswEntity::Pointer();
+54 -17
View File
@@ -17,6 +17,7 @@
#include <iostream>
#include <memory>
#include <mutex>
#include <shared_mutex>
#if defined(__linux__) || defined(__APPLE__)
#include <sys/mman.h>
#endif
@@ -246,19 +247,19 @@ class HnswStreamerEntity : public HnswEntity {
using NIHashMapPointer = std::shared_ptr<NIHashMap>;
//! Clone construct, used by clone method in subclasses
HnswStreamerEntity(IndexStreamer::Stats &stats, const HNSWHeader &hd,
size_t chunk_size, uint32_t node_index_mask_bits,
uint32_t upper_neighbor_mask_bits, bool filter_same_key,
bool get_vector_enabled,
const NIHashMapPointer &upper_neighbor_index,
std::shared_ptr<ailego::SharedMutex> &keys_map_lock,
const HashMapPointer<key_t, node_id_t> &keys_map,
bool use_key_info_map,
std::vector<Chunk::Pointer> &&node_chunks,
std::vector<Chunk::Pointer> &&upper_neighbor_chunks,
const ChunkBroker::Pointer &broker,
std::shared_ptr<std::vector<const uint8_t *>> node_bases,
std::shared_ptr<std::vector<const uint8_t *>> upper_bases)
HnswStreamerEntity(
IndexStreamer::Stats &stats, const HNSWHeader &hd, size_t chunk_size,
uint32_t node_index_mask_bits, uint32_t upper_neighbor_mask_bits,
bool filter_same_key, bool get_vector_enabled,
const NIHashMapPointer &upper_neighbor_index,
const std::shared_ptr<std::shared_mutex> &upper_neighbor_rw_mutex,
std::shared_ptr<ailego::SharedMutex> &keys_map_lock,
const HashMapPointer<key_t, node_id_t> &keys_map, bool use_key_info_map,
std::vector<Chunk::Pointer> &&node_chunks,
std::vector<Chunk::Pointer> &&upper_neighbor_chunks,
const ChunkBroker::Pointer &broker,
std::shared_ptr<std::vector<const uint8_t *>> node_bases,
std::shared_ptr<std::vector<const uint8_t *>> upper_bases)
: stats_(stats),
chunk_size_(chunk_size),
node_index_mask_bits_(node_index_mask_bits),
@@ -269,6 +270,7 @@ class HnswStreamerEntity : public HnswEntity {
filter_same_key_(filter_same_key),
get_vector_enabled_(get_vector_enabled),
use_key_info_map_(use_key_info_map),
upper_neighbor_rw_mutex_(upper_neighbor_rw_mutex),
upper_neighbor_index_(upper_neighbor_index),
keys_map_lock_(keys_map_lock),
keys_map_(keys_map),
@@ -323,6 +325,10 @@ class HnswStreamerEntity : public HnswEntity {
inline std::pair<uint32_t, uint32_t> get_upper_neighbor_chunk_loc(
level_t level, node_id_t id) const {
// Shared lock: concurrent readers are fine, but must synchronize with
// add_upper_neighbor's exclusive lock to avoid data-race on
// slots_.size() inside HnswIndexHashMap.
std::shared_lock<std::shared_mutex> lk(*upper_neighbor_rw_mutex_);
auto it = upper_neighbor_index_->find(id);
ailego_assert_abort(it != upper_neighbor_index_->end(),
"Get upper neighbor header failed");
@@ -370,6 +376,10 @@ class HnswStreamerEntity : public HnswEntity {
if (level == 0) {
return 0;
}
// Exclusive lock: protects upper_neighbor_chunks_.emplace_back() and
// upper_neighbor_index_->insert() from racing with concurrent find()
// calls in get_upper_neighbor_chunk_loc().
std::unique_lock<std::shared_mutex> lk(*upper_neighbor_rw_mutex_);
Chunk::Pointer chunk;
uint64_t chunk_offset = UINT64_MAX;
size_t neighbors_size = get_total_upper_neighbors_size(level);
@@ -408,14 +418,37 @@ class HnswStreamerEntity : public HnswEntity {
meta.level = level;
meta.index = (chunk_index << upper_neighbor_mask_bits_) |
(chunk_offset / upper_neighbor_size_);
size_t zero_start = chunk_offset;
chunk_offset += upper_neighbor_size_ * level;
if (ailego_unlikely(!upper_neighbor_index_->insert(id, meta.data))) {
LOG_ERROR("HashMap insert value failed");
// IMPORTANT: order matters here.
// 1) resize so the chunk's data_size covers the new region.
// 2) zero-fill the new region: storage backends like BufferStorage do
// NOT zero on resize -- only metadata is updated, and the underlying
// page may contain stale content from a previously-evicted page.
// Without this step, NeighborsHeader::neighbor_cnt is garbage and
// select_entry_point()/search_neighbors() iterate over garbage
// node_ids, eventually triggering find()'s assertion in
// get_upper_neighbor_chunk_loc().
// 3) ONLY THEN publish the entry to upper_neighbor_index_, so that any
// concurrent reader that finds this id already sees a properly
// zeroed upper-neighbor slot.
if (ailego_unlikely(chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", (size_t)chunk_offset);
return IndexError_Runtime;
}
if (ailego_unlikely(chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", (size_t)chunk_offset);
// Use std::vector instead of a VLA: VLAs are a GNU extension and may
// produce different codegen / be rejected under clang/MSVC.
std::vector<char> zeros(neighbors_size, 0);
if (ailego_unlikely(chunk->write(zero_start, zeros.data(),
neighbors_size) != neighbors_size)) {
LOG_ERROR("Chunk write zeros failed");
return IndexError_Runtime;
}
if (ailego_unlikely(!upper_neighbor_index_->insert(id, meta.data))) {
LOG_ERROR("HashMap insert value failed");
return IndexError_Runtime;
}
@@ -529,6 +562,10 @@ class HnswStreamerEntity : public HnswEntity {
protected:
IndexStreamer::Stats &stats_;
std::mutex mutex_{};
//! Guards upper_neighbor_index_ and upper_neighbor_chunks_ against
//! concurrent reads (find) and writes (insert/emplace_back).
//! Shared via shared_ptr so all clones synchronize on the SAME mutex.
mutable std::shared_ptr<std::shared_mutex> upper_neighbor_rw_mutex_{};
size_t max_index_size_{0UL};
uint32_t chunk_size_{kDefaultChunkSize};
uint32_t upper_neighbor_chunk_size_{kDefaultChunkSize};
@@ -41,9 +41,9 @@ class HnswIndexHashMap {
items_(reinterpret_cast<const Item *>(data)) {}
//! Return a empty loc or the key item loc
Slot(Chunk::Pointer &&chunk, IndexStorage::MemoryBlock &&mem_block)
: chunk_(std::move(chunk)), items_block_(std::move(mem_block)) {
items_ = reinterpret_cast<const Item *>(items_block_.data());
Slot(Chunk::Pointer &&chunk, std::vector<char> &&local_data)
: chunk_(std::move(chunk)), local_data_(std::move(local_data)) {
items_ = reinterpret_cast<const Item *>(local_data_.data());
}
const_iterator find(key_type key, uint32_t max_items, uint32_t mask) const {
auto it = &items_[key & mask];
@@ -73,8 +73,8 @@ class HnswIndexHashMap {
private:
Chunk::Pointer chunk_{};
const Item *items_{nullptr}; // point to chunk data
IndexStorage::MemoryBlock items_block_{};
const Item *items_{nullptr}; // point to local_data_
std::vector<char> local_data_{};
};
public:
@@ -179,14 +179,18 @@ class HnswIndexHashMap {
LOG_ERROR("Chunk resize failed, size=%zu", size);
return false;
}
//! Read the whole data to memory
IndexStorage::MemoryBlock data_block;
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
LOG_ERROR("Chunk read failed, size=%zu", size);
return false;
}
slots_.emplace_back(std::move(chunk), std::move(data_block));
//! Use a local zero-initialized buffer; new chunks contain all zeros,
//! so no buffer-pool read is needed and no ref_count is pinned.
//! NOTE: Previously this used `chunk->read(0U, data_block, size)` which
//! returns a view into the underlying BufferPool page. That made the
//! Slot's `items_` pointer alias buffer-pool memory shared across
//! threads, which under clang -O3 release exposed a data race on
//! Slot::find()'s probing read of `it->second` (concurrent
//! const_cast writes from insert() were not reliably visible). Using a
//! private zero-initialized vector matches the HNSW (non-RABITQ)
//! implementation and avoids this race.
std::vector<char> local_buf(size, 0);
slots_.emplace_back(std::move(chunk), std::move(local_buf));
return true;
}
@@ -208,13 +212,14 @@ class HnswIndexHashMap {
i, chunk->data_size(), size);
return IndexError_InvalidFormat;
}
//! Read the whole data to memory
IndexStorage::MemoryBlock data_block;
if (ailego_unlikely(chunk->read(0U, data_block, size) != size)) {
LOG_ERROR("Chunk read failed, size=%zu", size);
return false;
//! Copy chunk data into a local buffer via fetch() so that no
//! buffer-pool block is pinned for the lifetime of the Slot.
std::vector<char> local_buf(size);
if (ailego_unlikely(chunk->fetch(0U, local_buf.data(), size) != size)) {
LOG_ERROR("Chunk fetch failed, size=%zu", size);
return IndexError_InvalidFormat;
}
slots_.emplace_back(std::move(chunk), std::move(data_block));
slots_.emplace_back(std::move(chunk), std::move(local_buf));
}
return 0;
}
@@ -40,7 +40,7 @@ HnswRabitqStreamer::HnswRabitqStreamer(IndexProvider::Pointer provider,
provider_(std::move(provider)) {}
HnswRabitqStreamer::~HnswRabitqStreamer() {
if (state_ == STATE_INITED) {
if (state_ == STATE_INITED || state_ == STATE_OPENED) {
this->cleanup();
}
}
@@ -34,6 +34,7 @@ int HnswRabitqStreamerEntity::init(size_t max_doc_cnt) {
std::lock_guard<std::mutex> lock(mutex_);
broker_ = std::make_shared<HnswRabitqChunkBroker>(stats_);
upper_neighbor_index_ = std::make_shared<NIHashMap>();
upper_neighbor_rw_mutex_ = std::make_shared<std::shared_mutex>();
keys_map_lock_ = std::make_shared<ailego::SharedMutex>();
keys_map_ = std::make_shared<HashMap<key_t, node_id_t>>();
if (!keys_map_ || !upper_neighbor_index_ || !broker_ || !keys_map_lock_) {
@@ -697,8 +698,9 @@ const HnswRabitqEntity::Pointer HnswRabitqStreamerEntity::clone() const {
new (std::nothrow) HnswRabitqStreamerEntity(
stats_, header(), chunk_size_, node_index_mask_bits_,
upper_neighbor_mask_bits_, filter_same_key_, get_vector_enabled_,
upper_neighbor_index_, keys_map_lock_, keys_map_, use_key_info_map_,
std::move(node_chunks), std::move(upper_neighbor_chunks), broker_);
upper_neighbor_index_, upper_neighbor_rw_mutex_, keys_map_lock_,
keys_map_, use_key_info_map_, std::move(node_chunks),
std::move(upper_neighbor_chunks), broker_);
if (ailego_unlikely(!entity)) {
LOG_ERROR("HnswRabitqStreamerEntity new failed");
}
@@ -15,6 +15,7 @@
#pragma once
#include <iostream>
#include <shared_mutex>
#include <ailego/parallel/lock.h>
#include <sparsehash/dense_hash_map>
#include <sparsehash/dense_hash_set>
@@ -216,17 +217,17 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
using NIHashMapPointer = std::shared_ptr<NIHashMap>;
//! Private construct, only be called by clone method
HnswRabitqStreamerEntity(IndexStreamer::Stats &stats, const HNSWHeader &hd,
size_t chunk_size, uint32_t node_index_mask_bits,
uint32_t upper_neighbor_mask_bits,
bool filter_same_key, bool get_vector_enabled,
const NIHashMapPointer &upper_neighbor_index,
std::shared_ptr<ailego::SharedMutex> &keys_map_lock,
const HashMapPointer<key_t, node_id_t> &keys_map,
bool use_key_info_map,
std::vector<Chunk::Pointer> &&node_chunks,
std::vector<Chunk::Pointer> &&upper_neighbor_chunks,
const HnswRabitqChunkBroker::Pointer &broker)
HnswRabitqStreamerEntity(
IndexStreamer::Stats &stats, const HNSWHeader &hd, size_t chunk_size,
uint32_t node_index_mask_bits, uint32_t upper_neighbor_mask_bits,
bool filter_same_key, bool get_vector_enabled,
const NIHashMapPointer &upper_neighbor_index,
const std::shared_ptr<std::shared_mutex> &upper_neighbor_rw_mutex,
std::shared_ptr<ailego::SharedMutex> &keys_map_lock,
const HashMapPointer<key_t, node_id_t> &keys_map, bool use_key_info_map,
std::vector<Chunk::Pointer> &&node_chunks,
std::vector<Chunk::Pointer> &&upper_neighbor_chunks,
const HnswRabitqChunkBroker::Pointer &broker)
: stats_(stats),
chunk_size_(chunk_size),
node_index_mask_bits_(node_index_mask_bits),
@@ -237,6 +238,7 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
filter_same_key_(filter_same_key),
get_vector_enabled_(get_vector_enabled),
use_key_info_map_(use_key_info_map),
upper_neighbor_rw_mutex_(upper_neighbor_rw_mutex),
upper_neighbor_index_(upper_neighbor_index),
keys_map_lock_(keys_map_lock),
keys_map_(keys_map),
@@ -286,6 +288,11 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
inline std::pair<uint32_t, uint32_t> get_upper_neighbor_chunk_loc(
level_t level, node_id_t id) const {
// Shared lock: concurrent readers are fine, but must synchronize with
// add_upper_neighbor's exclusive lock to avoid data-race on
// slots_.size() inside HnswIndexHashMap (the emplace_back in alloc_slot
// is not atomic and concurrent find() may see a stale size value).
std::shared_lock<std::shared_mutex> lk(*upper_neighbor_rw_mutex_);
auto it = upper_neighbor_index_->find(id);
ailego_assert_abort(it != upper_neighbor_index_->end(),
"Get upper neighbor header failed");
@@ -334,6 +341,10 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
if (level == 0) {
return 0;
}
// Exclusive lock: protects upper_neighbor_chunks_.emplace_back() and
// upper_neighbor_index_->insert() from racing with concurrent find()
// calls in get_upper_neighbor_chunk_loc().
std::unique_lock<std::shared_mutex> lk(*upper_neighbor_rw_mutex_);
Chunk::Pointer chunk;
uint64_t chunk_offset = -1UL;
size_t neighbors_size = get_total_upper_neighbors_size(level);
@@ -373,14 +384,37 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
meta.level = level;
meta.index = (chunk_index << upper_neighbor_mask_bits_) |
(chunk_offset / upper_neighbor_size_);
size_t zero_start = chunk_offset;
chunk_offset += upper_neighbor_size_ * level;
if (ailego_unlikely(!upper_neighbor_index_->insert(id, meta.data))) {
LOG_ERROR("HashMap insert value failed");
// IMPORTANT: order matters here.
// 1) resize so the chunk's data_size covers the new region.
// 2) zero-fill the new region: storage backends like BufferStorage do
// NOT zero on resize -- only metadata is updated, and the underlying
// page may contain stale content from a previously-evicted page.
// Without this step, NeighborsHeader::neighbor_cnt is garbage and
// select_entry_point()/search_neighbors() iterate over garbage
// node_ids, eventually triggering find()'s assertion in
// get_upper_neighbor_chunk_loc() at line 291.
// 3) ONLY THEN publish the entry to upper_neighbor_index_, so that any
// concurrent reader that finds this id already sees a properly
// zeroed upper-neighbor slot.
if (ailego_unlikely(chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", (size_t)chunk_offset);
return IndexError_Runtime;
}
if (ailego_unlikely(chunk->resize(chunk_offset) != chunk_offset)) {
LOG_ERROR("Chunk resize to %zu failed", (size_t)chunk_offset);
// Use std::vector instead of a VLA: VLAs are a GNU extension and may
// produce different codegen / be rejected under clang/MSVC.
std::vector<char> zeros(neighbors_size, 0);
if (ailego_unlikely(chunk->write(zero_start, zeros.data(),
neighbors_size) != neighbors_size)) {
LOG_ERROR("Chunk write zeros failed");
return IndexError_Runtime;
}
if (ailego_unlikely(!upper_neighbor_index_->insert(id, meta.data))) {
LOG_ERROR("HashMap insert value failed");
return IndexError_Runtime;
}
@@ -503,6 +537,11 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
bool get_vector_enabled_{false};
bool use_key_info_map_{true};
// Shared via shared_ptr so that all cloned entities synchronize against
// the SAME mutex instance. A plain std::shared_mutex member would be
// independent per clone and provide no real protection for the shared
// upper_neighbor_index_ hashmap.
mutable std::shared_ptr<std::shared_mutex> upper_neighbor_rw_mutex_{};
NIHashMapPointer upper_neighbor_index_{};
mutable std::shared_ptr<ailego::SharedMutex> keys_map_lock_{};
@@ -27,7 +27,7 @@ namespace core {
HnswSparseStreamer::HnswSparseStreamer() : entity_(stats_) {}
HnswSparseStreamer::~HnswSparseStreamer() {
if (state_ == STATE_INITED) {
if (state_ == STATE_INITED || state_ == STATE_OPENED) {
this->cleanup();
}
}
+1 -1
View File
@@ -26,7 +26,7 @@ namespace core {
VamanaStreamer::VamanaStreamer() = default;
VamanaStreamer::~VamanaStreamer() {
if (state_ == STATE_INITED) {
if (state_ == STATE_INITED || state_ == STATE_OPENED) {
this->cleanup();
}
}