fix(vamana): medoid entry, concurrent build crash and prune quality. (#433)
* fix(vamana): medoid entry point, concurrent build crash and prune quality - Add DiskANN-standard medoid (centroid-closest point) as the persisted entry point, computed at dump time, replacing the fixed node-0 start. - Fix data races on node_chunks_ / dist_chunks_ / node_chunk_bases_ during concurrent build: add mutexes with double-checked locking and pre-reserve capacity so the lock-free read fast path never hits reallocation. - Fix RobustPrune for metrics with signed internal distance (quantized int8 cosine): introduce IndexMetric::build_distance_offset() to shift distances into a non-negative range, restoring a geometrically meaningful occlude_factor and improving graph quality / low-ef recall.
This commit is contained in:
@@ -265,6 +265,14 @@ void VamanaAlgorithm<EntityType>::robust_prune(node_id_t id,
|
||||
VamanaDistCalculator &dc = ctx->dist_calculator();
|
||||
size_t n = candidates.size();
|
||||
|
||||
// Build-time distance offset: shifts the internal distance into a
|
||||
// non-negative range so the ratio-based occlude_factor below is
|
||||
// geometrically meaningful. Zero for metrics whose internal distance is
|
||||
// already non-negative (SquaredEuclidean, etc.); 1.0 for the quantized int8
|
||||
// Cosine / NormalizedCosine path (internal distance is -cos, so
|
||||
// offset=1 yields 1-cos which matches DiskANN's normalized-L2 semantics).
|
||||
const float dist_offset = ctx->build_distance_offset();
|
||||
|
||||
// Truncate to max_occlusion_size (DiskANN's maxc parameter)
|
||||
size_t maxc = entity_.max_occlusion_size();
|
||||
if (maxc > 0 && n > maxc) {
|
||||
@@ -335,15 +343,21 @@ void VamanaAlgorithm<EntityType>::robust_prune(node_id_t id,
|
||||
// dist_between)
|
||||
// where dist_to_query = candidates[j].second (distance from query to j)
|
||||
// dist_between = batch_dists[k] (distance from selected to j)
|
||||
//
|
||||
// `dist_offset` shifts both distances into a non-negative range for
|
||||
// metrics whose internal distance can be negative (e.g. quantized
|
||||
// int8 cosine stores -cos(m,q) in [-1, 1]). Without this shift the
|
||||
// ratio loses its geometric meaning and RobustPrune produces a
|
||||
// poor-quality graph (low recall at low ef_search).
|
||||
for (uint32_t k = 0; k < batch_count; ++k) {
|
||||
uint32_t j = batch_indices[k];
|
||||
float dist_selected_to_candidate = batch_dists[k];
|
||||
if (dist_selected_to_candidate == 0.0f) {
|
||||
float dist_selected_to_candidate = batch_dists[k] + dist_offset;
|
||||
if (dist_selected_to_candidate <= 0.0f) {
|
||||
occlude_factor[j] = std::numeric_limits<float>::max();
|
||||
} else {
|
||||
occlude_factor[j] =
|
||||
std::max(occlude_factor[j],
|
||||
candidates[j].second / dist_selected_to_candidate);
|
||||
occlude_factor[j] = std::max(occlude_factor[j],
|
||||
(candidates[j].second + dist_offset) /
|
||||
dist_selected_to_candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,14 +24,22 @@ VamanaContext::VamanaContext(size_t dimension,
|
||||
: IndexContext(metric),
|
||||
entity_(entity),
|
||||
dc_(entity.get(), metric, dimension),
|
||||
metric_(metric) {}
|
||||
metric_(metric) {
|
||||
if (metric) {
|
||||
build_distance_offset_ = metric->build_distance_offset();
|
||||
}
|
||||
}
|
||||
|
||||
VamanaContext::VamanaContext(const IndexMetric::Pointer &metric,
|
||||
const VamanaEntity::Pointer &entity)
|
||||
: IndexContext(metric),
|
||||
entity_(entity),
|
||||
dc_(entity.get(), metric),
|
||||
metric_(metric) {}
|
||||
metric_(metric) {
|
||||
if (metric) {
|
||||
build_distance_offset_ = metric->build_distance_offset();
|
||||
}
|
||||
}
|
||||
|
||||
VamanaContext::~VamanaContext() {
|
||||
visit_filter_.destroy();
|
||||
@@ -99,6 +107,9 @@ int VamanaContext::update_context(ContextType type, const IndexMeta &meta,
|
||||
entity_ = entity;
|
||||
metric_ = metric;
|
||||
magic_ = magic_num;
|
||||
if (metric) {
|
||||
build_distance_offset_ = metric->build_distance_offset();
|
||||
}
|
||||
dc_.update(entity.get(), metric, meta.dimension());
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -149,6 +149,14 @@ class VamanaContext : public IndexContext {
|
||||
return batch_indices_buf_;
|
||||
}
|
||||
|
||||
//! Build-time distance offset cached from the metric. Used by RobustPrune
|
||||
//! to shift the internal distance to a non-negative range before computing
|
||||
//! the ratio-based occlude_factor. Zero for metrics whose internal distance
|
||||
//! is already non-negative (e.g. SquaredEuclidean).
|
||||
inline float build_distance_offset() const {
|
||||
return build_distance_offset_;
|
||||
}
|
||||
|
||||
inline void set_max_scan_num(uint32_t max_scan_num) {
|
||||
max_scan_num_ = max_scan_num;
|
||||
}
|
||||
@@ -307,6 +315,9 @@ class VamanaContext : public IndexContext {
|
||||
std::vector<float> batch_dists_buf_;
|
||||
std::vector<uint32_t> batch_indices_buf_;
|
||||
|
||||
//! Cached build-time distance offset (see build_distance_offset()).
|
||||
float build_distance_offset_{0.0f};
|
||||
|
||||
VisitFilter::Mode filter_mode_{VisitFilter::ByteMap};
|
||||
float filter_negative_prob_{VamanaEntity::kDefaultBFNegativeProbability};
|
||||
};
|
||||
|
||||
@@ -197,6 +197,17 @@ class VamanaEntity {
|
||||
header_.graph.entry_point = ep;
|
||||
}
|
||||
|
||||
// Calculate medoid (entry point) as the data point closest to the centroid
|
||||
// of all vectors, following DiskANN's standard approach.
|
||||
// Parameters:
|
||||
// dimension: vector dimension (number of elements per vector)
|
||||
// data_type: IndexMeta::DataType value (e.g. DT_FP32=2, DT_INT8=4,
|
||||
// DT_FP16=1)
|
||||
// Returns the medoid node ID, or kInvalidNodeId if no valid data.
|
||||
virtual node_id_t calculate_medoid(uint32_t dimension, uint32_t data_type) {
|
||||
return kInvalidNodeId;
|
||||
}
|
||||
|
||||
virtual int cleanup() {
|
||||
header_.clear();
|
||||
return 0;
|
||||
|
||||
@@ -341,6 +341,19 @@ int VamanaStreamer::dump(const IndexDumper::Pointer &dumper) {
|
||||
LOG_ERROR("Failed to serialize meta into dumper.");
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Calculate medoid (DiskANN standard: entry point = closest to centroid).
|
||||
// At dump time, data_type and dimension are fully known from meta_.
|
||||
if (entity_->doc_cnt() > 0) {
|
||||
node_id_t medoid = entity_->calculate_medoid(
|
||||
meta_.dimension(), static_cast<uint32_t>(meta_.data_type()));
|
||||
if (medoid != kInvalidNodeId && medoid != entity_->entry_point()) {
|
||||
LOG_INFO("Updating entry point from %u to medoid %u",
|
||||
entity_->entry_point(), medoid);
|
||||
entity_->update_entry_point(medoid);
|
||||
}
|
||||
}
|
||||
|
||||
return entity_->dump(dumper);
|
||||
}
|
||||
|
||||
|
||||
@@ -147,7 +147,10 @@ int VamanaStreamerEntity::add_vector(key_t key, const void *vec,
|
||||
}
|
||||
node_chunk = p.second;
|
||||
chunk_offset = 0UL;
|
||||
node_chunks_.emplace_back(node_chunk);
|
||||
{
|
||||
std::lock_guard<std::mutex> chunks_lock(node_chunks_mutex_);
|
||||
node_chunks_.emplace_back(node_chunk);
|
||||
}
|
||||
} else {
|
||||
node_chunk = node_chunks_[chunk_index];
|
||||
chunk_offset = node_chunk->data_size();
|
||||
@@ -211,7 +214,10 @@ int VamanaStreamerEntity::add_vector_with_id(node_id_t id, const void *vec) {
|
||||
return p.first;
|
||||
}
|
||||
node_chunk = p.second;
|
||||
node_chunks_.emplace_back(node_chunk);
|
||||
{
|
||||
std::lock_guard<std::mutex> chunks_lock(node_chunks_mutex_);
|
||||
node_chunks_.emplace_back(node_chunk);
|
||||
}
|
||||
}
|
||||
node_chunk = node_chunks_[chunk_idx];
|
||||
chunk_offset = (node_id & node_index_mask_) * node_size();
|
||||
@@ -631,6 +637,11 @@ int VamanaStreamerEntity::ensure_dist_storage() {
|
||||
|
||||
dist_entry_size_ = static_cast<uint32_t>(max_degree() * sizeof(dist_t));
|
||||
|
||||
// Pre-reserve dist_chunks_ to match node_chunks_ capacity so that
|
||||
// subsequent emplace_back in ensure_dist_chunk_for never triggers
|
||||
// reallocation while concurrent add_node threads read dist_chunks_.
|
||||
dist_chunks_.reserve(node_chunks_.capacity());
|
||||
|
||||
// Calculate how many dist chunks we need for existing nodes
|
||||
uint32_t total_docs = doc_cnt();
|
||||
if (total_docs == 0) {
|
||||
@@ -682,10 +693,17 @@ int VamanaStreamerEntity::ensure_dist_chunk_for(uint32_t chunk_index) {
|
||||
return dp.first;
|
||||
}
|
||||
dp.second->resize(dist_chunk_data_size);
|
||||
while (dist_chunks_.size() <= chunk_index) {
|
||||
dist_chunks_.emplace_back(nullptr);
|
||||
{
|
||||
// Protect dist_chunks_ modification against concurrent readers in
|
||||
// get_neighbor_dists/update_neighbor_dists (called from add_node without
|
||||
// mutex_). Uses node_chunks_mutex_ which is the same lock used by
|
||||
// sync_chunks for CHUNK_TYPE_NEIGHBOR_DIST.
|
||||
std::lock_guard<std::mutex> chunks_lock(node_chunks_mutex_);
|
||||
while (dist_chunks_.size() <= chunk_index) {
|
||||
dist_chunks_.emplace_back(nullptr);
|
||||
}
|
||||
dist_chunks_[chunk_index] = std::move(dp.second);
|
||||
}
|
||||
dist_chunks_[chunk_index] = std::move(dp.second);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -769,5 +787,122 @@ void VamanaStreamerEntity::set_neighbor_dist(node_id_t id, uint32_t idx,
|
||||
dist_chunks_[loc.first]->write(offset, &dist, sizeof(dist_t));
|
||||
}
|
||||
|
||||
// calculate_medoid: Compute the medoid (entry point) following DiskANN's
|
||||
// standard approach:
|
||||
// 1. Compute the centroid (component-wise mean) of all vectors in float
|
||||
// space.
|
||||
// 2. Find the data point closest to this centroid using squared L2 distance
|
||||
// in float space.
|
||||
// 3. Return that point's node ID as the medoid.
|
||||
//
|
||||
// Called at dump time to set the optimal entry point for the persisted index.
|
||||
// data_type uses IndexMeta::DataType values: DT_FP16=1, DT_FP32=2, DT_INT8=4.
|
||||
// ============================================================================
|
||||
node_id_t VamanaStreamerEntity::calculate_medoid(uint32_t dimension,
|
||||
uint32_t data_type) {
|
||||
uint32_t n = doc_cnt();
|
||||
if (n == 0) return kInvalidNodeId;
|
||||
if (dimension == 0) return kInvalidNodeId;
|
||||
|
||||
// data_type constants matching IndexMeta::DataType
|
||||
constexpr uint32_t DT_FP16 = 1;
|
||||
constexpr uint32_t DT_FP32 = 2;
|
||||
constexpr uint32_t DT_INT8 = 4;
|
||||
|
||||
if (data_type != DT_FP32 && data_type != DT_INT8 && data_type != DT_FP16) {
|
||||
LOG_WARN("calculate_medoid: unsupported data_type=%u, skip", data_type);
|
||||
return entry_point();
|
||||
}
|
||||
|
||||
// Step 1: Compute centroid (mean) of all vectors in float space.
|
||||
std::vector<float> centroid(dimension, 0.0f);
|
||||
uint32_t valid_count = 0;
|
||||
|
||||
for (node_id_t i = 0; i < n; ++i) {
|
||||
if (get_key(i) == kInvalidKey) continue;
|
||||
const void *vec = get_vector(i);
|
||||
if (vec == nullptr) continue;
|
||||
|
||||
switch (data_type) {
|
||||
case DT_FP32: {
|
||||
const float *fv = static_cast<const float *>(vec);
|
||||
for (uint32_t d = 0; d < dimension; ++d) centroid[d] += fv[d];
|
||||
break;
|
||||
}
|
||||
case DT_INT8: {
|
||||
const int8_t *iv = static_cast<const int8_t *>(vec);
|
||||
for (uint32_t d = 0; d < dimension; ++d)
|
||||
centroid[d] += static_cast<float>(iv[d]);
|
||||
break;
|
||||
}
|
||||
case DT_FP16: {
|
||||
const uint16_t *hv = static_cast<const uint16_t *>(vec);
|
||||
for (uint32_t d = 0; d < dimension; ++d)
|
||||
centroid[d] += ailego::FloatHelper::ToFP32(hv[d]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
valid_count++;
|
||||
}
|
||||
|
||||
if (valid_count == 0) return kInvalidNodeId;
|
||||
if (valid_count == 1) {
|
||||
for (node_id_t i = 0; i < n; ++i) {
|
||||
if (get_key(i) != kInvalidKey && get_vector(i) != nullptr) return i;
|
||||
}
|
||||
return kInvalidNodeId;
|
||||
}
|
||||
|
||||
float inv = 1.0f / static_cast<float>(valid_count);
|
||||
for (uint32_t d = 0; d < dimension; ++d) centroid[d] *= inv;
|
||||
|
||||
// Step 2: Find the data point closest to the centroid (squared L2).
|
||||
node_id_t medoid = kInvalidNodeId;
|
||||
float min_dist = std::numeric_limits<float>::max();
|
||||
|
||||
for (node_id_t i = 0; i < n; ++i) {
|
||||
if (get_key(i) == kInvalidKey) continue;
|
||||
const void *vec = get_vector(i);
|
||||
if (vec == nullptr) continue;
|
||||
|
||||
float dist = 0.0f;
|
||||
switch (data_type) {
|
||||
case DT_FP32: {
|
||||
const float *fv = static_cast<const float *>(vec);
|
||||
for (uint32_t d = 0; d < dimension; ++d) {
|
||||
float diff = fv[d] - centroid[d];
|
||||
dist += diff * diff;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case DT_INT8: {
|
||||
const int8_t *iv = static_cast<const int8_t *>(vec);
|
||||
for (uint32_t d = 0; d < dimension; ++d) {
|
||||
float diff = static_cast<float>(iv[d]) - centroid[d];
|
||||
dist += diff * diff;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case DT_FP16: {
|
||||
const uint16_t *hv = static_cast<const uint16_t *>(vec);
|
||||
for (uint32_t d = 0; d < dimension; ++d) {
|
||||
float diff = ailego::FloatHelper::ToFP32(hv[d]) - centroid[d];
|
||||
dist += diff * diff;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (dist < min_dist) {
|
||||
min_dist = dist;
|
||||
medoid = i;
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INFO("Calculated medoid: node_id=%u, min_sq_dist=%.4f, valid_docs=%u",
|
||||
medoid, min_dist, valid_count);
|
||||
return medoid;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
|
||||
@@ -67,6 +67,10 @@ class VamanaStreamerEntity : public VamanaEntity {
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
void update_entry_point(node_id_t ep) override;
|
||||
|
||||
// Calculate medoid: find the data point closest to the centroid
|
||||
// of all vectors (DiskANN standard entry point selection).
|
||||
node_id_t calculate_medoid(uint32_t dimension, uint32_t data_type) override;
|
||||
|
||||
// --- Neighbor distance storage ---
|
||||
int ensure_dist_storage() override;
|
||||
bool dist_storage_loaded() const override {
|
||||
@@ -182,13 +186,18 @@ class VamanaStreamerEntity : public VamanaEntity {
|
||||
}
|
||||
|
||||
//! Lazy chunk synchronization: fetches chunks from broker when needed.
|
||||
//! Each clone entity has its own node_chunks_ vector, so concurrent
|
||||
//! search threads do not race with the writer's emplace_back.
|
||||
//! Protected by node_chunks_mutex_ to synchronize with add_vector's
|
||||
//! emplace_back during concurrent build.
|
||||
void sync_chunks(ChunkBroker::CHUNK_TYPE type, size_t idx,
|
||||
std::vector<Chunk::Pointer> *chunks) const {
|
||||
if (ailego_likely(idx < chunks->size())) {
|
||||
return;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(node_chunks_mutex_);
|
||||
// Double-check after acquiring lock
|
||||
if (idx < chunks->size()) {
|
||||
return;
|
||||
}
|
||||
for (size_t i = chunks->size(); i <= idx; ++i) {
|
||||
auto chunk = broker_->get_chunk(type, i);
|
||||
ailego_assert_with(!!chunk, "get chunk failed");
|
||||
@@ -307,6 +316,9 @@ class VamanaStreamerEntity : public VamanaEntity {
|
||||
|
||||
ChunkBroker::Pointer broker_;
|
||||
|
||||
//! Protects node_chunks_ against concurrent emplace_back from add_vector
|
||||
//! (writer) and sync_chunks from greedy_search (reader threads during build).
|
||||
mutable std::mutex node_chunks_mutex_{};
|
||||
mutable std::vector<Chunk::Pointer> node_chunks_{};
|
||||
|
||||
private:
|
||||
@@ -504,6 +516,17 @@ class VamanaMmapStreamerEntity : public VamanaStreamerEntity {
|
||||
}
|
||||
|
||||
void sync_node_chunk_bases(uint32_t chunk_idx) const {
|
||||
std::lock_guard<std::mutex> lock(chunk_bases_mutex_);
|
||||
// Double-check after acquiring lock to avoid redundant sync
|
||||
if (chunk_idx < node_chunk_bases_.size()) {
|
||||
return;
|
||||
}
|
||||
// Pre-reserve to match node_chunks_ capacity so that subsequent
|
||||
// push_back never triggers reallocation — the lock-free fast path in
|
||||
// get_node_chunk_base reads existing elements without holding the mutex.
|
||||
if (node_chunk_bases_.capacity() < node_chunks_.capacity()) {
|
||||
node_chunk_bases_.reserve(node_chunks_.capacity());
|
||||
}
|
||||
sync_node_chunks(chunk_idx);
|
||||
const auto &chunks = node_chunks_;
|
||||
for (size_t i = node_chunk_bases_.size(); i <= chunk_idx; ++i) {
|
||||
@@ -513,6 +536,7 @@ class VamanaMmapStreamerEntity : public VamanaStreamerEntity {
|
||||
}
|
||||
}
|
||||
|
||||
mutable std::mutex chunk_bases_mutex_{};
|
||||
mutable std::vector<const char *> node_chunk_bases_{};
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user