refactor: remove hamming metric (#365)

* refactor: remove binary metric

* fix: fix windows
This commit is contained in:
rayx
2026-05-18 19:31:20 +08:00
committed by GitHub
parent 273ffc2ad8
commit 3e02031eb8
19 changed files with 5 additions and 3359 deletions
@@ -17,7 +17,6 @@
#include <ailego/algorithm/kmeans.h>
#include <gtest/gtest.h>
#include <zvec/ailego/container/params.h>
#include <zvec/ailego/internal/platform.h>
#include "zvec/core/framework/index_framework.h"
using namespace zvec::core;
@@ -209,106 +208,6 @@ TEST(OptKmeansCluster, General) {
// }
// }
TEST(OptKmeansCluster, BinaryGeneral) {
// Prepare index data
const uint32_t count = 5000u;
const uint32_t dimension = 1024u;
IndexMeta index_meta;
index_meta.set_meta(IndexMeta::DataType::DT_BINARY32, dimension);
index_meta.set_metric("SquaredEuclidean", 0, Params());
std::shared_ptr<CompactIndexFeatures> features(
new CompactIndexFeatures(index_meta));
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(0.0, 1.0);
for (uint32_t i = 0; i < count; ++i) {
BinaryVector<uint32_t> vec(dimension);
for (size_t j = 0; j < dimension; ++j) {
if (dist(gen) >= 0.5) {
vec.set(j);
}
}
features->emplace(vec.data());
}
std::cout << "---------- FIRST ----------\n";
// Create a Kmeans cluster
IndexCluster::Pointer cluster =
IndexFactory::CreateCluster("OptKmeansCluster");
ASSERT_TRUE(!!cluster);
Params params;
params.set("proxima.general.cluster.count", 1);
params.set("proxima.optkmeans.cluster.count", 56);
ASSERT_EQ(0, cluster->init(index_meta, params));
ASSERT_EQ(0, cluster->mount(features));
cluster->suggest(64u);
auto threads = std::make_shared<SingleQueueIndexThreads>();
std::cout << "---------- FIRST ----------\n";
std::vector<IndexCluster::Centroid> centroids;
std::vector<uint32_t> labels;
ASSERT_NE(0, cluster->classify(threads, centroids));
ASSERT_NE(0, cluster->label(threads, centroids, &labels));
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<uint32_t>();
unsigned int mask = 0x1;
std::cout << it.follows() << " (" << it.score() << ") { "
<< !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", "
<< !!(vec[0] & (mask << 2)) << ", ... , "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 2))) << ", "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 1))) << " }"
<< std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- SECOND ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<uint32_t>();
unsigned int mask = 0x1;
std::cout << it.follows() << " (" << it.score() << ") { "
<< !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", "
<< !!(vec[0] & (mask << 2)) << ", ... , "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 2))) << ", "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 1))) << " }"
<< std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- THIRD ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<uint32_t>();
unsigned int mask = 0x1;
std::cout << it.follows() << " (" << it.score() << ") { "
<< !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", "
<< !!(vec[0] & (mask << 2)) << ", ... , "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 2))) << ", "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 1))) << " }"
<< std::endl;
ASSERT_EQ(0u, it.similars().size());
}
ASSERT_EQ(0, cluster->classify(threads, centroids));
ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
}
TEST(OptKmeansCluster, IN4General) {
// Prepare index data
const uint32_t count = 5000u;
@@ -118,18 +118,6 @@ TEST_F(FlatBuilderTest, TestInt8WithRandomDimension) {
ASSERT_EQ(0, builder->init(meta_, params));
}
TEST_F(FlatBuilderTest, TestBinaryInvalidColumnMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim + 2);
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "TestGeneral";
}
TEST_F(FlatBuilderTest, TestBuildWithRowMajor) {
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_ROW);
@@ -185,35 +173,6 @@ TEST_F(FlatBuilderTest, TestInt8BuildWithRowMajor) {
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestBinaryBuildWithRowMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
meta_.set_major_order(IndexMeta::MO_ROW);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
BinaryVector<uint32_t> vec(dim);
for (size_t j = 0; j < dim && j < i; ++j) {
vec.set(j);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
int ret = builder->train(holder);
EXPECT_EQ(0, ret);
ret = builder->build(holder);
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestBuildWithColumnMajor) {
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
meta_.set_metric("SquaredEuclidean", 0, Params());
@@ -270,35 +229,6 @@ TEST_F(FlatBuilderTest, TestInt8BuildWithColumnMajor) {
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestBinaryBuildWithColumnMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
BinaryVector<uint32_t> vec(dim);
for (size_t j = 0; j < dim && j < i; ++j) {
vec.set(j);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
int ret = builder->train(holder);
EXPECT_EQ(0, ret);
ret = builder->build(holder);
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestWithRowMajor) {
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
meta_.set_metric("SquaredEuclidean", 0, Params());
@@ -349,32 +279,6 @@ TEST_F(FlatBuilderTest, TestInt8WithRowMajor) {
ASSERT_EQ(0, builder->cleanup());
}
TEST_F(FlatBuilderTest, TestBinaryWithRowMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
meta_.set_major_order(IndexMeta::MO_ROW);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
std::string path = dir_ + "TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
BinaryVector<uint32_t> vec(dim);
for (size_t j = 0; j < dim && j < i; ++j) {
vec.set(j);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
build_process(builder, holder);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
}
TEST_F(FlatBuilderTest, TestWithColumnMajor) {
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
meta_.set_metric("SquaredEuclidean", 0, Params());
@@ -425,32 +329,6 @@ TEST_F(FlatBuilderTest, TestInt8WithColumnMajor) {
ASSERT_EQ(0, builder->cleanup());
}
TEST_F(FlatBuilderTest, TestBinaryWithColumnMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
std::string path = dir_ + "TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
BinaryVector<uint32_t> vec(dim);
for (size_t j = 0; j < dim && j < i; ++j) {
vec.set(j);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
build_process(builder, holder);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
}
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
@@ -3447,93 +3447,6 @@ TEST_F(HnswStreamerTest, TestGroupInBruteforceSearch) {
}
}
#if 0
TEST_F(HnswStreamerTest, TestBinaryConverter) {
uint32_t dimension = 2560;
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("HnswStreamer");
ASSERT_TRUE(streamer != nullptr);
ailego::Params params;
// params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10);
// params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 16);
// params.set(PARAM_HNSW_STREAMER_EFCONSTRUCTION, 10);
// params.set(PARAM_HNSW_STREAMER_EF, 5);
params.set(PARAM_HNSW_STREAMER_BRUTE_FORCE_THRESHOLD, 1000U);
ailego::Params stg_params;
IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, dimension);
index_meta_raw.set_metric("InnerProduct", 0, ailego::Params());
ailego::Params converter_params;
auto converter = IndexFactory::CreateConverter("BinaryConverter");
ASSERT_TRUE(converter != nullptr);
converter->init(index_meta_raw, converter_params);
IndexMeta index_meta = converter->meta();
auto reformer = IndexFactory::CreateReformer(index_meta.reformer_name());
ASSERT_TRUE(reformer != nullptr);
ASSERT_EQ(0, reformer->init(index_meta.reformer_params()));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "TestBinaryConverter.index", true));
ASSERT_EQ(0, streamer->init(index_meta, params));
ASSERT_EQ(0, streamer->open(storage));
size_t cnt = 5000U;
auto ctx = streamer->create_context();
ASSERT_TRUE(!!ctx);
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dimension);
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(-2.0, 2.0);
std::vector<NumericalVector<float>> vecs;
for (size_t i = 0; i < cnt; i++) {
NumericalVector<float> vec(dimension);
for (size_t j = 0; j < dimension; ++j) {
vec[j] = dist(gen);
}
std::string new_vec;
IndexQueryMeta new_meta;
ASSERT_EQ(0, reformer->convert(vec.data(), qmeta, &new_vec, &new_meta));
ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx));
vecs.push_back(vec);
}
size_t query_cnt = 200U;
auto knnCtx = streamer->create_context();
float epison = 1e-6;
for (size_t i = 0; i < query_cnt; i++) {
auto &vec = vecs[i];
std::string new_query;
IndexQueryMeta new_meta;
ASSERT_EQ(0, reformer->transform(vec.data(), qmeta, &new_query, &new_meta));
size_t topk = 50;
knnCtx->set_topk(topk);
ASSERT_EQ(0, streamer->search_impl(new_query.data(), new_meta, knnCtx));
auto &results = knnCtx->result();
ASSERT_EQ(topk, results.size());
ASSERT_EQ(i, results[0].key());
ASSERT_NEAR(0, results[0].score(), epison);
}
}
#endif
TEST_F(HnswStreamerTest, TestAddAndSearchWithID) {
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("HnswStreamer");
@@ -3647,131 +3560,6 @@ TEST_F(HnswStreamerTest, TestAddAndSearchWithID) {
// EXPECT_GT(cost, 2.0f);
}
#if 0
TEST_F(HnswStreamerTest, TestBasicRefiner) {
uint32_t dimension = 1120;
IndexStreamer::Pointer base_streamer =
IndexFactory::CreateStreamer("HnswStreamer");
ASSERT_TRUE(base_streamer != nullptr);
IndexStreamer::Pointer refine_streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_TRUE(refine_streamer != nullptr);
IndexRefiner::Pointer refiner = IndexFactory::CreateRefiner("BasicRefiner");
ASSERT_TRUE(refiner != nullptr);
ailego::Params params;
IndexMeta index_meta(IndexMeta::DataType::DT_FP32, dimension);
index_meta.set_metric("InnerProduct", 0, ailego::Params());
ailego::Params converter_params;
auto converter = IndexFactory::CreateConverter("BinaryConverter");
ASSERT_TRUE(converter != nullptr);
converter->init(index_meta, converter_params);
IndexMeta index_meta_binary = converter->meta();
auto reformer =
IndexFactory::CreateReformer(index_meta_binary.reformer_name());
ASSERT_TRUE(reformer != nullptr);
ASSERT_EQ(0, reformer->init(index_meta_binary.reformer_params()));
// base streamer
ailego::Params base_stg_params;
auto base_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, base_storage->init(base_stg_params));
ASSERT_EQ(0, base_storage->open(dir_ + "TestBasicRefinerBase.index", true));
ASSERT_EQ(0, base_streamer->init(index_meta_binary, params));
ASSERT_EQ(0, base_streamer->open(base_storage));
auto base_ctx = base_streamer->create_context();
ASSERT_TRUE(!!base_ctx);
// refine streamer
ailego::Params refine_stg_params;
auto refine_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, refine_storage->init(refine_stg_params));
ASSERT_EQ(0,
refine_storage->open(dir_ + "TestBasicRefinerRefine.index", true));
ASSERT_EQ(0, refine_streamer->init(index_meta, params));
ASSERT_EQ(0, refine_streamer->open(refine_storage));
auto refine_ctx = refine_streamer->create_context();
ASSERT_TRUE(!!refine_ctx);
ailego::Params refiner_params;
ASSERT_EQ(0, refiner->init(base_streamer, refine_streamer, refiner_params));
auto ctx = refiner->create_context();
ASSERT_TRUE(!!ctx);
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dimension);
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(-2.0, 2.0);
std::vector<NumericalVector<float>> vecs;
size_t cnt = 5000U;
for (size_t i = 0; i < cnt; i++) {
NumericalVector<float> vec(dimension);
for (size_t j = 0; j < dimension; ++j) {
vec[j] = dist(gen);
}
std::string binary_vec;
IndexQueryMeta binary_qmeta;
ASSERT_EQ(0,
reformer->convert(vec.data(), qmeta, &binary_vec, &binary_qmeta));
ASSERT_EQ(0, refiner->add_impl(i, binary_vec.data(), binary_qmeta,
vec.data(), qmeta, ctx));
vecs.push_back(vec);
}
size_t query_cnt = 200U;
// size_t query_cnt = 1U;
auto searcherCtx = refiner->create_context();
for (size_t i = 0; i < query_cnt; i++) {
auto &vec = vecs[i];
// float abs_value{0};
// for (size_t j = 0; j < dimension; ++j) {
// std::cout << "dim: " << j << ", value: " << vec[j] << std::endl;
// abs_value += std::abs(vec[j]);
// }
// std::cout << "abs value: " << abs_value << std::endl;
std::string new_query;
IndexQueryMeta binary_qmeta;
ASSERT_EQ(
0, reformer->transform(vec.data(), qmeta, &new_query, &binary_qmeta));
size_t topk = 50;
searcherCtx->set_topk(topk);
ASSERT_EQ(0, refiner->search_impl(new_query.data(), binary_qmeta,
vec.data(), qmeta, searcherCtx));
auto &results = searcherCtx->result();
ASSERT_EQ(topk, results.size());
ASSERT_EQ(i, results[0].key());
// for (size_t i = 0; i < results.size(); ++i) {
// std::cout << i << ", id: " << results[i].index()
// << ", score: " << results[i].score() << std::endl;
// }
}
}
#endif
TEST_F(HnswStreamerTest, TestContiguousMemorySearch) {
// Build index with mmap mode
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
@@ -1405,258 +1405,6 @@ TEST_F(IVFSearcherTest, TestRowMajorInt8WithBuildMemory) {
EXPECT_EQ(0, ret);
}
TEST_F(IVFSearcherTest, TestColumnMajorBinaryWithBuildMemory) {
IVFBuilder builder;
dimension_ = 256;
index_meta_.set_meta(IndexMeta::DataType::DT_BINARY32, dimension_);
index_meta_.set_metric("Hamming", 0, Params());
index_meta_.set_major_order(IndexMeta::MO_COLUMN);
int ret = builder.init(index_meta_, params_);
EXPECT_EQ(0, ret);
size_t fnum = 257;
prepare_binary_index_holder(0, fnum);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(0, ret);
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("FileDumper");
ret = dumper->create(index_path_);
EXPECT_EQ(0, ret);
ret = builder.dump(dumper);
EXPECT_EQ(0, dumper->close());
EXPECT_EQ((size_t)fnum, builder.stats().built_count());
EXPECT_EQ((size_t)fnum, builder.stats().dumped_count());
EXPECT_EQ((size_t)0, builder.stats().discarded_count());
IVFSearcher searcher;
Params params;
params.set(PARAM_IVF_SEARCHER_SCAN_RATIO, 1.0);
params.set(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD, 1);
ret = searcher.init(params);
EXPECT_EQ(0, ret);
IndexStorage::Pointer container =
IndexFactory::CreateStorage("MMapFileReadStorage");
EXPECT_TRUE(!!container);
Params container_params;
container_params.set("proxima.mmap_file.container.memory_warmup", true);
container->init(container_params);
ret = container->open(index_path_, false);
EXPECT_EQ(0, ret);
ret = searcher.load(container, IndexMetric::Pointer());
EXPECT_EQ(0, ret);
BinaryVector<uint32_t> query(dimension_);
for (size_t i = 0; i < dimension_; ++i) {
query.set(i);
}
size_t qnum = 63;
BinaryVector<uint32_t> query1(dimension_ * qnum);
for (size_t i = 0; i < qnum; ++i) {
for (size_t j = 0; j < dimension_ && j < i; ++j) {
query1.set(i * dimension_ + j);
}
}
auto context = searcher.create_context();
IndexQueryMeta qmeta(IndexMeta::DataType::DT_BINARY32, dimension_);
// single bf serch
{
size_t topk = 128;
context->set_topk(topk);
ret = searcher.search_bf_impl(query.data(), qmeta, context);
EXPECT_EQ(0, ret);
const IndexDocumentList &result = context->result(0);
EXPECT_EQ((size_t)topk, result.size());
for (size_t i = 0; i < topk; ++i) {
EXPECT_EQ((uint64_t)256 - i, result[i].key());
EXPECT_FLOAT_EQ((float)i, result[i].score());
}
}
// batch bf serch
{
size_t topk = 1;
context->set_topk(topk);
ret = searcher.search_bf_impl(query1.data(), qmeta, qnum, context);
EXPECT_EQ(0, ret);
for (size_t q = 0; q < qnum; ++q) {
const IndexDocumentList &result = context->result(q);
EXPECT_EQ((size_t)topk, result.size());
EXPECT_EQ((uint64_t)q, result[0].key());
EXPECT_FLOAT_EQ((float)0, result[0].score());
}
}
// single knn search
{
size_t topk = 100;
context->set_topk(topk);
ret = searcher.search_impl(query.data(), qmeta, context);
EXPECT_EQ(0, ret);
const IndexDocumentList &result = context->result(0);
EXPECT_EQ((size_t)topk, result.size());
for (size_t i = 0; i < topk; ++i) {
EXPECT_EQ((uint64_t)256 - i, result[i].key());
EXPECT_FLOAT_EQ((float)i, result[i].score());
}
}
// batch knn serch
{
size_t topk = 1;
context->set_topk(topk);
ret = searcher.search_impl(query1.data(), qmeta, qnum, context);
EXPECT_EQ(0, ret);
for (size_t q = 0; q < qnum; ++q) {
const IndexDocumentList &result = context->result(q);
EXPECT_EQ((size_t)topk, result.size());
EXPECT_EQ((uint64_t)q, result[0].key());
EXPECT_FLOAT_EQ((float)0, result[0].score());
}
}
ret = searcher.unload();
EXPECT_EQ(0, ret);
}
TEST_F(IVFSearcherTest, TestRowMajorBinaryWithBuildMemory) {
IVFBuilder builder;
dimension_ = 256;
index_meta_.set_meta(IndexMeta::DataType::DT_BINARY32, dimension_);
index_meta_.set_metric("Hamming", 0, Params());
index_meta_.set_major_order(IndexMeta::MO_ROW);
int ret = builder.init(index_meta_, params_);
EXPECT_EQ(0, ret);
size_t fnum = 257;
prepare_binary_index_holder(0, fnum);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(0, ret);
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("FileDumper");
ret = dumper->create(index_path_);
EXPECT_EQ(0, ret);
ret = builder.dump(dumper);
EXPECT_EQ(0, dumper->close());
EXPECT_EQ((size_t)fnum, builder.stats().built_count());
EXPECT_EQ((size_t)fnum, builder.stats().dumped_count());
EXPECT_EQ((size_t)0, builder.stats().discarded_count());
IVFSearcher searcher;
Params params;
params.set(PARAM_IVF_SEARCHER_SCAN_RATIO, 1.0);
params.set(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD, 1);
ret = searcher.init(params);
EXPECT_EQ(0, ret);
IndexStorage::Pointer container =
IndexFactory::CreateStorage("MMapFileReadStorage");
EXPECT_TRUE(!!container);
Params container_params;
container_params.set("proxima.mmap_file.container.memory_warmup", true);
container->init(container_params);
ret = container->open(index_path_, false);
EXPECT_EQ(0, ret);
ret = searcher.load(container, IndexMetric::Pointer());
EXPECT_EQ(0, ret);
BinaryVector<uint32_t> query(dimension_);
for (size_t i = 0; i < dimension_; ++i) {
query.set(i);
}
size_t qnum = 63;
BinaryVector<uint32_t> query1(dimension_ * qnum);
for (size_t i = 0; i < qnum; ++i) {
for (size_t j = 0; j < dimension_ && j < i; ++j) {
query1.set(i * dimension_ + j);
}
}
auto context = searcher.create_context();
IndexQueryMeta qmeta(IndexMeta::DataType::DT_BINARY32, dimension_);
// single bf serch
{
size_t topk = 128;
context->set_topk(topk);
ret = searcher.search_bf_impl(query.data(), qmeta, context);
EXPECT_EQ(0, ret);
const IndexDocumentList &result = context->result(0);
EXPECT_EQ((size_t)topk, result.size());
for (size_t i = 0; i < topk; ++i) {
EXPECT_EQ((uint64_t)256 - i, result[i].key());
EXPECT_FLOAT_EQ((float)i, result[i].score());
}
}
// batch bf serch
{
size_t topk = 1;
context->set_topk(topk);
ret = searcher.search_bf_impl(query1.data(), qmeta, qnum, context);
EXPECT_EQ(0, ret);
for (size_t q = 0; q < qnum; ++q) {
const IndexDocumentList &result = context->result(q);
EXPECT_EQ((size_t)topk, result.size());
EXPECT_EQ((uint64_t)q, result[0].key());
EXPECT_FLOAT_EQ((float)0, result[0].score());
}
}
// single knn search
{
size_t topk = 100;
context->set_topk(topk);
ret = searcher.search_impl(query.data(), qmeta, context);
EXPECT_EQ(0, ret);
const IndexDocumentList &result = context->result(0);
EXPECT_EQ((size_t)topk, result.size());
for (size_t i = 0; i < topk; ++i) {
EXPECT_EQ((uint64_t)256 - i, result[i].key());
EXPECT_FLOAT_EQ((float)i, result[i].score());
}
}
// batch knn serch
{
size_t topk = 1;
context->set_topk(topk);
ret = searcher.search_impl(query1.data(), qmeta, qnum, context);
EXPECT_EQ(0, ret);
for (size_t q = 0; q < qnum; ++q) {
const IndexDocumentList &result = context->result(q);
EXPECT_EQ((size_t)topk, result.size());
EXPECT_EQ((uint64_t)q, result[0].key());
EXPECT_FLOAT_EQ((float)0, result[0].score());
}
}
ret = searcher.unload();
EXPECT_EQ(0, ret);
}
TEST_F(IVFSearcherTest, TestSearchWithEmptyCentroid) {
IVFBuilder builder;
Params params;