diff --git a/src/core/algorithm/vamana/vamana_algorithm.cc b/src/core/algorithm/vamana/vamana_algorithm.cc index dbbff34..a57d36b 100644 --- a/src/core/algorithm/vamana/vamana_algorithm.cc +++ b/src/core/algorithm/vamana/vamana_algorithm.cc @@ -265,6 +265,14 @@ void VamanaAlgorithm::robust_prune(node_id_t id, VamanaDistCalculator &dc = ctx->dist_calculator(); size_t n = candidates.size(); + // Build-time distance offset: shifts the internal distance into a + // non-negative range so the ratio-based occlude_factor below is + // geometrically meaningful. Zero for metrics whose internal distance is + // already non-negative (SquaredEuclidean, etc.); 1.0 for the quantized int8 + // Cosine / NormalizedCosine path (internal distance is -cos, so + // offset=1 yields 1-cos which matches DiskANN's normalized-L2 semantics). + const float dist_offset = ctx->build_distance_offset(); + // Truncate to max_occlusion_size (DiskANN's maxc parameter) size_t maxc = entity_.max_occlusion_size(); if (maxc > 0 && n > maxc) { @@ -335,15 +343,21 @@ void VamanaAlgorithm::robust_prune(node_id_t id, // dist_between) // where dist_to_query = candidates[j].second (distance from query to j) // dist_between = batch_dists[k] (distance from selected to j) + // + // `dist_offset` shifts both distances into a non-negative range for + // metrics whose internal distance can be negative (e.g. quantized + // int8 cosine stores -cos(m,q) in [-1, 1]). Without this shift the + // ratio loses its geometric meaning and RobustPrune produces a + // poor-quality graph (low recall at low ef_search). for (uint32_t k = 0; k < batch_count; ++k) { uint32_t j = batch_indices[k]; - float dist_selected_to_candidate = batch_dists[k]; - if (dist_selected_to_candidate == 0.0f) { + float dist_selected_to_candidate = batch_dists[k] + dist_offset; + if (dist_selected_to_candidate <= 0.0f) { occlude_factor[j] = std::numeric_limits::max(); } else { - occlude_factor[j] = - std::max(occlude_factor[j], - candidates[j].second / dist_selected_to_candidate); + occlude_factor[j] = std::max(occlude_factor[j], + (candidates[j].second + dist_offset) / + dist_selected_to_candidate); } } } diff --git a/src/core/algorithm/vamana/vamana_context.cc b/src/core/algorithm/vamana/vamana_context.cc index a3832e1..a1483ec 100644 --- a/src/core/algorithm/vamana/vamana_context.cc +++ b/src/core/algorithm/vamana/vamana_context.cc @@ -24,14 +24,22 @@ VamanaContext::VamanaContext(size_t dimension, : IndexContext(metric), entity_(entity), dc_(entity.get(), metric, dimension), - metric_(metric) {} + metric_(metric) { + if (metric) { + build_distance_offset_ = metric->build_distance_offset(); + } +} VamanaContext::VamanaContext(const IndexMetric::Pointer &metric, const VamanaEntity::Pointer &entity) : IndexContext(metric), entity_(entity), dc_(entity.get(), metric), - metric_(metric) {} + metric_(metric) { + if (metric) { + build_distance_offset_ = metric->build_distance_offset(); + } +} VamanaContext::~VamanaContext() { visit_filter_.destroy(); @@ -99,6 +107,9 @@ int VamanaContext::update_context(ContextType type, const IndexMeta &meta, entity_ = entity; metric_ = metric; magic_ = magic_num; + if (metric) { + build_distance_offset_ = metric->build_distance_offset(); + } dc_.update(entity.get(), metric, meta.dimension()); return 0; } diff --git a/src/core/algorithm/vamana/vamana_context.h b/src/core/algorithm/vamana/vamana_context.h index b1d3a16..38cc845 100644 --- a/src/core/algorithm/vamana/vamana_context.h +++ b/src/core/algorithm/vamana/vamana_context.h @@ -149,6 +149,14 @@ class VamanaContext : public IndexContext { return batch_indices_buf_; } + //! Build-time distance offset cached from the metric. Used by RobustPrune + //! to shift the internal distance to a non-negative range before computing + //! the ratio-based occlude_factor. Zero for metrics whose internal distance + //! is already non-negative (e.g. SquaredEuclidean). + inline float build_distance_offset() const { + return build_distance_offset_; + } + inline void set_max_scan_num(uint32_t max_scan_num) { max_scan_num_ = max_scan_num; } @@ -307,6 +315,9 @@ class VamanaContext : public IndexContext { std::vector batch_dists_buf_; std::vector batch_indices_buf_; + //! Cached build-time distance offset (see build_distance_offset()). + float build_distance_offset_{0.0f}; + VisitFilter::Mode filter_mode_{VisitFilter::ByteMap}; float filter_negative_prob_{VamanaEntity::kDefaultBFNegativeProbability}; }; diff --git a/src/core/algorithm/vamana/vamana_entity.h b/src/core/algorithm/vamana/vamana_entity.h index 1efff0e..be06bf2 100644 --- a/src/core/algorithm/vamana/vamana_entity.h +++ b/src/core/algorithm/vamana/vamana_entity.h @@ -197,6 +197,17 @@ class VamanaEntity { header_.graph.entry_point = ep; } + // Calculate medoid (entry point) as the data point closest to the centroid + // of all vectors, following DiskANN's standard approach. + // Parameters: + // dimension: vector dimension (number of elements per vector) + // data_type: IndexMeta::DataType value (e.g. DT_FP32=2, DT_INT8=4, + // DT_FP16=1) + // Returns the medoid node ID, or kInvalidNodeId if no valid data. + virtual node_id_t calculate_medoid(uint32_t dimension, uint32_t data_type) { + return kInvalidNodeId; + } + virtual int cleanup() { header_.clear(); return 0; diff --git a/src/core/algorithm/vamana/vamana_streamer.cc b/src/core/algorithm/vamana/vamana_streamer.cc index 2738a98..d03163c 100644 --- a/src/core/algorithm/vamana/vamana_streamer.cc +++ b/src/core/algorithm/vamana/vamana_streamer.cc @@ -341,6 +341,19 @@ int VamanaStreamer::dump(const IndexDumper::Pointer &dumper) { LOG_ERROR("Failed to serialize meta into dumper."); return ret; } + + // Calculate medoid (DiskANN standard: entry point = closest to centroid). + // At dump time, data_type and dimension are fully known from meta_. + if (entity_->doc_cnt() > 0) { + node_id_t medoid = entity_->calculate_medoid( + meta_.dimension(), static_cast(meta_.data_type())); + if (medoid != kInvalidNodeId && medoid != entity_->entry_point()) { + LOG_INFO("Updating entry point from %u to medoid %u", + entity_->entry_point(), medoid); + entity_->update_entry_point(medoid); + } + } + return entity_->dump(dumper); } diff --git a/src/core/algorithm/vamana/vamana_streamer_entity.cc b/src/core/algorithm/vamana/vamana_streamer_entity.cc index ffe8e75..b9bb381 100644 --- a/src/core/algorithm/vamana/vamana_streamer_entity.cc +++ b/src/core/algorithm/vamana/vamana_streamer_entity.cc @@ -147,7 +147,10 @@ int VamanaStreamerEntity::add_vector(key_t key, const void *vec, } node_chunk = p.second; chunk_offset = 0UL; - node_chunks_.emplace_back(node_chunk); + { + std::lock_guard chunks_lock(node_chunks_mutex_); + node_chunks_.emplace_back(node_chunk); + } } else { node_chunk = node_chunks_[chunk_index]; chunk_offset = node_chunk->data_size(); @@ -211,7 +214,10 @@ int VamanaStreamerEntity::add_vector_with_id(node_id_t id, const void *vec) { return p.first; } node_chunk = p.second; - node_chunks_.emplace_back(node_chunk); + { + std::lock_guard chunks_lock(node_chunks_mutex_); + node_chunks_.emplace_back(node_chunk); + } } node_chunk = node_chunks_[chunk_idx]; chunk_offset = (node_id & node_index_mask_) * node_size(); @@ -631,6 +637,11 @@ int VamanaStreamerEntity::ensure_dist_storage() { dist_entry_size_ = static_cast(max_degree() * sizeof(dist_t)); + // Pre-reserve dist_chunks_ to match node_chunks_ capacity so that + // subsequent emplace_back in ensure_dist_chunk_for never triggers + // reallocation while concurrent add_node threads read dist_chunks_. + dist_chunks_.reserve(node_chunks_.capacity()); + // Calculate how many dist chunks we need for existing nodes uint32_t total_docs = doc_cnt(); if (total_docs == 0) { @@ -682,10 +693,17 @@ int VamanaStreamerEntity::ensure_dist_chunk_for(uint32_t chunk_index) { return dp.first; } dp.second->resize(dist_chunk_data_size); - while (dist_chunks_.size() <= chunk_index) { - dist_chunks_.emplace_back(nullptr); + { + // Protect dist_chunks_ modification against concurrent readers in + // get_neighbor_dists/update_neighbor_dists (called from add_node without + // mutex_). Uses node_chunks_mutex_ which is the same lock used by + // sync_chunks for CHUNK_TYPE_NEIGHBOR_DIST. + std::lock_guard chunks_lock(node_chunks_mutex_); + while (dist_chunks_.size() <= chunk_index) { + dist_chunks_.emplace_back(nullptr); + } + dist_chunks_[chunk_index] = std::move(dp.second); } - dist_chunks_[chunk_index] = std::move(dp.second); return 0; } @@ -769,5 +787,122 @@ void VamanaStreamerEntity::set_neighbor_dist(node_id_t id, uint32_t idx, dist_chunks_[loc.first]->write(offset, &dist, sizeof(dist_t)); } +// calculate_medoid: Compute the medoid (entry point) following DiskANN's +// standard approach: +// 1. Compute the centroid (component-wise mean) of all vectors in float +// space. +// 2. Find the data point closest to this centroid using squared L2 distance +// in float space. +// 3. Return that point's node ID as the medoid. +// +// Called at dump time to set the optimal entry point for the persisted index. +// data_type uses IndexMeta::DataType values: DT_FP16=1, DT_FP32=2, DT_INT8=4. +// ============================================================================ +node_id_t VamanaStreamerEntity::calculate_medoid(uint32_t dimension, + uint32_t data_type) { + uint32_t n = doc_cnt(); + if (n == 0) return kInvalidNodeId; + if (dimension == 0) return kInvalidNodeId; + + // data_type constants matching IndexMeta::DataType + constexpr uint32_t DT_FP16 = 1; + constexpr uint32_t DT_FP32 = 2; + constexpr uint32_t DT_INT8 = 4; + + if (data_type != DT_FP32 && data_type != DT_INT8 && data_type != DT_FP16) { + LOG_WARN("calculate_medoid: unsupported data_type=%u, skip", data_type); + return entry_point(); + } + + // Step 1: Compute centroid (mean) of all vectors in float space. + std::vector centroid(dimension, 0.0f); + uint32_t valid_count = 0; + + for (node_id_t i = 0; i < n; ++i) { + if (get_key(i) == kInvalidKey) continue; + const void *vec = get_vector(i); + if (vec == nullptr) continue; + + switch (data_type) { + case DT_FP32: { + const float *fv = static_cast(vec); + for (uint32_t d = 0; d < dimension; ++d) centroid[d] += fv[d]; + break; + } + case DT_INT8: { + const int8_t *iv = static_cast(vec); + for (uint32_t d = 0; d < dimension; ++d) + centroid[d] += static_cast(iv[d]); + break; + } + case DT_FP16: { + const uint16_t *hv = static_cast(vec); + for (uint32_t d = 0; d < dimension; ++d) + centroid[d] += ailego::FloatHelper::ToFP32(hv[d]); + break; + } + } + valid_count++; + } + + if (valid_count == 0) return kInvalidNodeId; + if (valid_count == 1) { + for (node_id_t i = 0; i < n; ++i) { + if (get_key(i) != kInvalidKey && get_vector(i) != nullptr) return i; + } + return kInvalidNodeId; + } + + float inv = 1.0f / static_cast(valid_count); + for (uint32_t d = 0; d < dimension; ++d) centroid[d] *= inv; + + // Step 2: Find the data point closest to the centroid (squared L2). + node_id_t medoid = kInvalidNodeId; + float min_dist = std::numeric_limits::max(); + + for (node_id_t i = 0; i < n; ++i) { + if (get_key(i) == kInvalidKey) continue; + const void *vec = get_vector(i); + if (vec == nullptr) continue; + + float dist = 0.0f; + switch (data_type) { + case DT_FP32: { + const float *fv = static_cast(vec); + for (uint32_t d = 0; d < dimension; ++d) { + float diff = fv[d] - centroid[d]; + dist += diff * diff; + } + break; + } + case DT_INT8: { + const int8_t *iv = static_cast(vec); + for (uint32_t d = 0; d < dimension; ++d) { + float diff = static_cast(iv[d]) - centroid[d]; + dist += diff * diff; + } + break; + } + case DT_FP16: { + const uint16_t *hv = static_cast(vec); + for (uint32_t d = 0; d < dimension; ++d) { + float diff = ailego::FloatHelper::ToFP32(hv[d]) - centroid[d]; + dist += diff * diff; + } + break; + } + } + + if (dist < min_dist) { + min_dist = dist; + medoid = i; + } + } + + LOG_INFO("Calculated medoid: node_id=%u, min_sq_dist=%.4f, valid_docs=%u", + medoid, min_dist, valid_count); + return medoid; +} + } // namespace core } // namespace zvec diff --git a/src/core/algorithm/vamana/vamana_streamer_entity.h b/src/core/algorithm/vamana/vamana_streamer_entity.h index 9ca498d..59652f2 100644 --- a/src/core/algorithm/vamana/vamana_streamer_entity.h +++ b/src/core/algorithm/vamana/vamana_streamer_entity.h @@ -67,6 +67,10 @@ class VamanaStreamerEntity : public VamanaEntity { int dump(const IndexDumper::Pointer &dumper) override; void update_entry_point(node_id_t ep) override; + // Calculate medoid: find the data point closest to the centroid + // of all vectors (DiskANN standard entry point selection). + node_id_t calculate_medoid(uint32_t dimension, uint32_t data_type) override; + // --- Neighbor distance storage --- int ensure_dist_storage() override; bool dist_storage_loaded() const override { @@ -182,13 +186,18 @@ class VamanaStreamerEntity : public VamanaEntity { } //! Lazy chunk synchronization: fetches chunks from broker when needed. - //! Each clone entity has its own node_chunks_ vector, so concurrent - //! search threads do not race with the writer's emplace_back. + //! Protected by node_chunks_mutex_ to synchronize with add_vector's + //! emplace_back during concurrent build. void sync_chunks(ChunkBroker::CHUNK_TYPE type, size_t idx, std::vector *chunks) const { if (ailego_likely(idx < chunks->size())) { return; } + std::lock_guard lock(node_chunks_mutex_); + // Double-check after acquiring lock + if (idx < chunks->size()) { + return; + } for (size_t i = chunks->size(); i <= idx; ++i) { auto chunk = broker_->get_chunk(type, i); ailego_assert_with(!!chunk, "get chunk failed"); @@ -307,6 +316,9 @@ class VamanaStreamerEntity : public VamanaEntity { ChunkBroker::Pointer broker_; + //! Protects node_chunks_ against concurrent emplace_back from add_vector + //! (writer) and sync_chunks from greedy_search (reader threads during build). + mutable std::mutex node_chunks_mutex_{}; mutable std::vector node_chunks_{}; private: @@ -504,6 +516,17 @@ class VamanaMmapStreamerEntity : public VamanaStreamerEntity { } void sync_node_chunk_bases(uint32_t chunk_idx) const { + std::lock_guard lock(chunk_bases_mutex_); + // Double-check after acquiring lock to avoid redundant sync + if (chunk_idx < node_chunk_bases_.size()) { + return; + } + // Pre-reserve to match node_chunks_ capacity so that subsequent + // push_back never triggers reallocation — the lock-free fast path in + // get_node_chunk_base reads existing elements without holding the mutex. + if (node_chunk_bases_.capacity() < node_chunks_.capacity()) { + node_chunk_bases_.reserve(node_chunks_.capacity()); + } sync_node_chunks(chunk_idx); const auto &chunks = node_chunks_; for (size_t i = node_chunk_bases_.size(); i <= chunk_idx; ++i) { @@ -513,6 +536,7 @@ class VamanaMmapStreamerEntity : public VamanaStreamerEntity { } } + mutable std::mutex chunk_bases_mutex_{}; mutable std::vector node_chunk_bases_{}; }; diff --git a/src/core/metric/quantized_integer_metric.cc b/src/core/metric/quantized_integer_metric.cc index 3829502..1d46f77 100644 --- a/src/core/metric/quantized_integer_metric.cc +++ b/src/core/metric/quantized_integer_metric.cc @@ -267,6 +267,19 @@ class QuantizedIntegerMetric : public IndexMetric { origin_metric_type_ == MetricType::kCosine; } + //! Build-time distance offset to make the internal distance non-negative. + //! For kCosine / kNormalizedCosine on the quantized int8 path, the internal + //! distance is -cos(m,q) in [-1, 1]. Adding 1.0 maps it to [0, 2] (i.e. + //! 1 - cos), which is what ratio-based pruning (Vamana RobustPrune) needs + //! for a geometrically meaningful occlude_factor. + float build_distance_offset(void) const override { + if (origin_metric_type_ == MetricType::kCosine || + origin_metric_type_ == MetricType::kNormalizedCosine) { + return 1.0f; + } + return 0.0f; + } + //! Retrieve query metric object of this index metric Pointer query_metric(void) const override { if (origin_metric_type_ == MetricType::kMipsSquaredEuclidean) { diff --git a/src/include/zvec/core/framework/index_metric.h b/src/include/zvec/core/framework/index_metric.h index ad34ba2..2fc3c8f 100644 --- a/src/include/zvec/core/framework/index_metric.h +++ b/src/include/zvec/core/framework/index_metric.h @@ -137,6 +137,19 @@ struct IndexMetric : public IndexModule { virtual DistanceBatchQueryPreprocessFunc get_query_preprocess_func() const { return nullptr; } + + //! Distance offset applied during graph build to make the internal distance + //! non-negative for ratio-based pruning (e.g. Vamana RobustPrune's + //! occlude_factor = d(q,c) / d(p,c)). Metrics whose internal distance is a + //! well-defined non-negative value (SquaredEuclidean, 1-cos, etc.) should + //! leave the default 0. Metrics that internally store a signed quantity + //! (e.g. -cos for Cosine / NormalizedCosine on the quantized int8 path) + //! should override this to return a constant C such that (internal_dist + C) + //! is always non-negative and preserves the ordering of the original + //! distance. + virtual float build_distance_offset(void) const { + return 0.0f; + } }; } // namespace core diff --git a/tests/core/algorithm/vamana/vamana_streamer_test.cc b/tests/core/algorithm/vamana/vamana_streamer_test.cc index b9eed4b..cdae281 100644 --- a/tests/core/algorithm/vamana/vamana_streamer_test.cc +++ b/tests/core/algorithm/vamana/vamana_streamer_test.cc @@ -713,6 +713,78 @@ TEST_F(VamanaStreamerTest, TestKnnConcurrentAddAndSearch) { searchFuture.wait(); } +// Test concurrent build (parallel add_impl) which was crashing due to +// unprotected node_chunks_ / node_chunk_bases_ access during chunk allocation. +TEST_F(VamanaStreamerTest, TestConcurrentBuild) { + constexpr size_t dim = kDim; + constexpr size_t total_vectors = 5000; + constexpr size_t thread_count = 4; + + ailego::Params params; + params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 32U); + params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 100U); + params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f); + params.set(PARAM_VAMANA_STREAMER_EF, 64U); + params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 500U); + params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 50U * 1024U * 1024U); + + IndexMeta meta(IndexMeta::DataType::DT_FP32, dim); + meta.set_metric("SquaredEuclidean", 0, ailego::Params()); + + auto streamer = IndexFactory::CreateStreamer("VamanaStreamer"); + ASSERT_NE(nullptr, streamer); + ASSERT_EQ(0, streamer->init(meta, params)); + + auto storage = IndexFactory::CreateStorage("MMapFileStorage"); + ASSERT_NE(nullptr, storage); + ailego::Params stg_params; + ASSERT_EQ(0, storage->init(stg_params)); + ASSERT_EQ(0, storage->open(dir_ + "TestConcurrentBuild", true)); + ASSERT_EQ(0, streamer->open(storage)); + + // Parallel insertion from multiple threads (mimics local_builder behavior) + std::atomic error_count{0}; + std::vector> futures; + + for (size_t t = 0; t < thread_count; ++t) { + futures.push_back(std::async(std::launch::async, [&, t]() { + auto ctx = streamer->create_context(); + ASSERT_TRUE(!!ctx); + IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim); + NumericalVector vec(dim); + + for (size_t i = t; i < total_vectors; i += thread_count) { + for (size_t j = 0; j < dim; ++j) { + vec[j] = static_cast(i) + static_cast(j) * 0.01f; + } + int ret = streamer->add_impl(i, vec.data(), qmeta, ctx); + if (ret != 0) { + error_count.fetch_add(1); + return; + } + } + })); + } + + for (auto &f : futures) { + f.wait(); + } + ASSERT_EQ(0, error_count.load()); + + // Verify search still works correctly after concurrent build + auto search_ctx = streamer->create_context(); + ASSERT_TRUE(!!search_ctx); + search_ctx->set_topk(1); + IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim); + NumericalVector vec(dim); + for (size_t j = 0; j < dim; ++j) { + vec[j] = 0.0f; + } + ASSERT_EQ(0, streamer->search_impl(vec.data(), qmeta, search_ctx)); + auto &result = search_ctx->result(); + ASSERT_GT(result.size(), 0UL); +} + } // namespace core } // namespace zvec