fix(diskann, hnsw sparse builder): lost-wakeup race in builder progress loops (#530)

This commit is contained in:
Jalin Wang
2026-06-26 16:35:24 +08:00
committed by GitHub
parent 0a008ebfbf
commit cdef8686c7
5 changed files with 115 additions and 40 deletions
@@ -16,6 +16,7 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <chrono>
#include <future>
#include <gtest/gtest.h>
#include <zvec/ailego/container/vector.h>
@@ -99,6 +100,49 @@ TEST_F(DiskAnnBuilderTest, TestGeneral) {
ASSERT_GT(stats.built_costtime(), 0UL);
}
// Regression test: building a small DiskAnn index must complete quickly.
// A lost-wakeup bug in the condition-variable progress loops previously caused
// 15–30 second stalls during train/build on small datasets because
// notify_one() was either missing or racing against a wrong predicate.
TEST_F(DiskAnnBuilderTest, SmallDatasetBuildTime) {
constexpr size_t kSmallDim = 4;
constexpr size_t kSmallDocCnt = 12;
auto meta = make_shared<IndexMeta>(IndexMeta::DataType::DT_FP32, kSmallDim);
meta->set_metric("SquaredEuclidean", 0, Params());
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
ASSERT_NE(builder, nullptr);
auto holder = make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
kSmallDim);
for (size_t i = 0; i < kSmallDocCnt; ++i) {
NumericalVector<float> vec(kSmallDim, static_cast<float>(i));
ASSERT_TRUE(holder->emplace(i, vec));
}
Params params;
params.set("zvec.diskann.builder.max_degree", 32);
params.set("zvec.diskann.builder.list_size", 50);
params.set("zvec.diskann.builder.max_pq_chunk_num", 2);
params.set("zvec.diskann.builder.threads", 4);
ASSERT_EQ(0, builder->init(*meta, params));
auto t0 = std::chrono::steady_clock::now();
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto t1 = std::chrono::steady_clock::now();
auto elapsed_ms =
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
// Before the fix, this took 15–30 seconds. After the fix, it should
// complete in well under 5 seconds even on slow CI machines.
EXPECT_LT(elapsed_ms, 5000)
<< "DiskAnn build with " << kSmallDocCnt << " vectors took " << elapsed_ms
<< " ms — likely a lost-wakeup regression in progress loops.";
}
// DiskAnn is now exposed implicitly: no caller ever invokes a
// ``LoadDiskAnnPlugin`` / ``IsLibAioAvailable`` API (those were removed from
// the public surface together with ``zvec.load_diskann_plugin()`` in Python).