refactor: remove hamming metric (#365)
* refactor: remove binary metric * fix: fix windows
This commit is contained in:
@@ -17,7 +17,6 @@
|
||||
#include <ailego/algorithm/kmeans.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include <zvec/ailego/container/params.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include "zvec/core/framework/index_framework.h"
|
||||
|
||||
using namespace zvec::core;
|
||||
@@ -209,106 +208,6 @@ TEST(OptKmeansCluster, General) {
|
||||
// }
|
||||
// }
|
||||
|
||||
TEST(OptKmeansCluster, BinaryGeneral) {
|
||||
// Prepare index data
|
||||
const uint32_t count = 5000u;
|
||||
const uint32_t dimension = 1024u;
|
||||
|
||||
IndexMeta index_meta;
|
||||
index_meta.set_meta(IndexMeta::DataType::DT_BINARY32, dimension);
|
||||
index_meta.set_metric("SquaredEuclidean", 0, Params());
|
||||
|
||||
std::shared_ptr<CompactIndexFeatures> features(
|
||||
new CompactIndexFeatures(index_meta));
|
||||
|
||||
std::random_device rd;
|
||||
std::mt19937 gen(rd());
|
||||
std::uniform_real_distribution<float> dist(0.0, 1.0);
|
||||
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
BinaryVector<uint32_t> vec(dimension);
|
||||
for (size_t j = 0; j < dimension; ++j) {
|
||||
if (dist(gen) >= 0.5) {
|
||||
vec.set(j);
|
||||
}
|
||||
}
|
||||
features->emplace(vec.data());
|
||||
}
|
||||
|
||||
std::cout << "---------- FIRST ----------\n";
|
||||
|
||||
// Create a Kmeans cluster
|
||||
IndexCluster::Pointer cluster =
|
||||
IndexFactory::CreateCluster("OptKmeansCluster");
|
||||
ASSERT_TRUE(!!cluster);
|
||||
|
||||
Params params;
|
||||
params.set("proxima.general.cluster.count", 1);
|
||||
params.set("proxima.optkmeans.cluster.count", 56);
|
||||
|
||||
ASSERT_EQ(0, cluster->init(index_meta, params));
|
||||
ASSERT_EQ(0, cluster->mount(features));
|
||||
cluster->suggest(64u);
|
||||
|
||||
auto threads = std::make_shared<SingleQueueIndexThreads>();
|
||||
|
||||
std::cout << "---------- FIRST ----------\n";
|
||||
std::vector<IndexCluster::Centroid> centroids;
|
||||
std::vector<uint32_t> labels;
|
||||
ASSERT_NE(0, cluster->classify(threads, centroids));
|
||||
ASSERT_NE(0, cluster->label(threads, centroids, &labels));
|
||||
ASSERT_EQ(0, cluster->cluster(threads, centroids));
|
||||
|
||||
for (const auto &it : centroids) {
|
||||
const auto &vec = it.vector<uint32_t>();
|
||||
|
||||
unsigned int mask = 0x1;
|
||||
std::cout << it.follows() << " (" << it.score() << ") { "
|
||||
<< !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", "
|
||||
<< !!(vec[0] & (mask << 2)) << ", ... , "
|
||||
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 2))) << ", "
|
||||
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 1))) << " }"
|
||||
<< std::endl;
|
||||
ASSERT_EQ(0u, it.similars().size());
|
||||
}
|
||||
|
||||
std::cout << "---------- SECOND ----------\n";
|
||||
ASSERT_EQ(0, cluster->cluster(threads, centroids));
|
||||
|
||||
for (const auto &it : centroids) {
|
||||
const auto &vec = it.vector<uint32_t>();
|
||||
|
||||
unsigned int mask = 0x1;
|
||||
std::cout << it.follows() << " (" << it.score() << ") { "
|
||||
<< !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", "
|
||||
<< !!(vec[0] & (mask << 2)) << ", ... , "
|
||||
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 2))) << ", "
|
||||
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 1))) << " }"
|
||||
<< std::endl;
|
||||
ASSERT_EQ(0u, it.similars().size());
|
||||
}
|
||||
|
||||
std::cout << "---------- THIRD ----------\n";
|
||||
ASSERT_EQ(0, cluster->cluster(threads, centroids));
|
||||
|
||||
for (const auto &it : centroids) {
|
||||
const auto &vec = it.vector<uint32_t>();
|
||||
|
||||
unsigned int mask = 0x1;
|
||||
std::cout << it.follows() << " (" << it.score() << ") { "
|
||||
<< !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", "
|
||||
<< !!(vec[0] & (mask << 2)) << ", ... , "
|
||||
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 2))) << ", "
|
||||
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 1))) << " }"
|
||||
<< std::endl;
|
||||
ASSERT_EQ(0u, it.similars().size());
|
||||
}
|
||||
|
||||
ASSERT_EQ(0, cluster->classify(threads, centroids));
|
||||
ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
|
||||
}
|
||||
|
||||
|
||||
TEST(OptKmeansCluster, IN4General) {
|
||||
// Prepare index data
|
||||
const uint32_t count = 5000u;
|
||||
|
||||
@@ -118,18 +118,6 @@ TEST_F(FlatBuilderTest, TestInt8WithRandomDimension) {
|
||||
ASSERT_EQ(0, builder->init(meta_, params));
|
||||
}
|
||||
|
||||
TEST_F(FlatBuilderTest, TestBinaryInvalidColumnMajor) {
|
||||
size_t dim = (DIMENSION + 31) / 32 * 32;
|
||||
meta_.set_metric("Hamming", 0, Params());
|
||||
meta_.set_meta(IndexMeta::DT_BINARY32, dim + 2);
|
||||
meta_.set_major_order(IndexMeta::MO_COLUMN);
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
Params params;
|
||||
ASSERT_EQ(0, builder->init(meta_, params));
|
||||
std::string path = dir_ + "TestGeneral";
|
||||
}
|
||||
|
||||
TEST_F(FlatBuilderTest, TestBuildWithRowMajor) {
|
||||
meta_.set_metric("SquaredEuclidean", 0, Params());
|
||||
meta_.set_major_order(IndexMeta::MO_ROW);
|
||||
@@ -185,35 +173,6 @@ TEST_F(FlatBuilderTest, TestInt8BuildWithRowMajor) {
|
||||
EXPECT_EQ(0, ret);
|
||||
}
|
||||
|
||||
TEST_F(FlatBuilderTest, TestBinaryBuildWithRowMajor) {
|
||||
size_t dim = (DIMENSION + 31) / 32 * 32;
|
||||
meta_.set_metric("Hamming", 0, Params());
|
||||
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
|
||||
meta_.set_major_order(IndexMeta::MO_ROW);
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
Params params;
|
||||
ASSERT_EQ(0, builder->init(meta_, params));
|
||||
std::string path = dir_ + "TestGeneral";
|
||||
|
||||
auto holder =
|
||||
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
|
||||
size_t doc_cnt = 128UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
BinaryVector<uint32_t> vec(dim);
|
||||
for (size_t j = 0; j < dim && j < i; ++j) {
|
||||
vec.set(j);
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
|
||||
int ret = builder->train(holder);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
ret = builder->build(holder);
|
||||
EXPECT_EQ(0, ret);
|
||||
}
|
||||
|
||||
TEST_F(FlatBuilderTest, TestBuildWithColumnMajor) {
|
||||
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
|
||||
meta_.set_metric("SquaredEuclidean", 0, Params());
|
||||
@@ -270,35 +229,6 @@ TEST_F(FlatBuilderTest, TestInt8BuildWithColumnMajor) {
|
||||
EXPECT_EQ(0, ret);
|
||||
}
|
||||
|
||||
TEST_F(FlatBuilderTest, TestBinaryBuildWithColumnMajor) {
|
||||
size_t dim = (DIMENSION + 31) / 32 * 32;
|
||||
meta_.set_metric("Hamming", 0, Params());
|
||||
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
|
||||
meta_.set_major_order(IndexMeta::MO_COLUMN);
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
Params params;
|
||||
ASSERT_EQ(0, builder->init(meta_, params));
|
||||
std::string path = dir_ + "TestGeneral";
|
||||
|
||||
auto holder =
|
||||
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
|
||||
size_t doc_cnt = 128UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
BinaryVector<uint32_t> vec(dim);
|
||||
for (size_t j = 0; j < dim && j < i; ++j) {
|
||||
vec.set(j);
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
|
||||
int ret = builder->train(holder);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
ret = builder->build(holder);
|
||||
EXPECT_EQ(0, ret);
|
||||
}
|
||||
|
||||
TEST_F(FlatBuilderTest, TestWithRowMajor) {
|
||||
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
|
||||
meta_.set_metric("SquaredEuclidean", 0, Params());
|
||||
@@ -349,32 +279,6 @@ TEST_F(FlatBuilderTest, TestInt8WithRowMajor) {
|
||||
ASSERT_EQ(0, builder->cleanup());
|
||||
}
|
||||
|
||||
TEST_F(FlatBuilderTest, TestBinaryWithRowMajor) {
|
||||
size_t dim = (DIMENSION + 31) / 32 * 32;
|
||||
meta_.set_metric("Hamming", 0, Params());
|
||||
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
|
||||
meta_.set_major_order(IndexMeta::MO_ROW);
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
Params params;
|
||||
std::string path = dir_ + "TestGeneral";
|
||||
|
||||
auto holder =
|
||||
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
|
||||
size_t doc_cnt = 128UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
BinaryVector<uint32_t> vec(dim);
|
||||
for (size_t j = 0; j < dim && j < i; ++j) {
|
||||
vec.set(j);
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
build_process(builder, holder);
|
||||
|
||||
// cleanup and rebuild
|
||||
ASSERT_EQ(0, builder->cleanup());
|
||||
}
|
||||
|
||||
TEST_F(FlatBuilderTest, TestWithColumnMajor) {
|
||||
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
|
||||
meta_.set_metric("SquaredEuclidean", 0, Params());
|
||||
@@ -425,32 +329,6 @@ TEST_F(FlatBuilderTest, TestInt8WithColumnMajor) {
|
||||
ASSERT_EQ(0, builder->cleanup());
|
||||
}
|
||||
|
||||
TEST_F(FlatBuilderTest, TestBinaryWithColumnMajor) {
|
||||
size_t dim = (DIMENSION + 31) / 32 * 32;
|
||||
meta_.set_metric("Hamming", 0, Params());
|
||||
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
|
||||
meta_.set_major_order(IndexMeta::MO_COLUMN);
|
||||
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
|
||||
ASSERT_NE(builder, nullptr);
|
||||
Params params;
|
||||
std::string path = dir_ + "TestGeneral";
|
||||
|
||||
auto holder =
|
||||
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
|
||||
size_t doc_cnt = 128UL;
|
||||
for (size_t i = 0; i < doc_cnt; i++) {
|
||||
BinaryVector<uint32_t> vec(dim);
|
||||
for (size_t j = 0; j < dim && j < i; ++j) {
|
||||
vec.set(j);
|
||||
}
|
||||
ASSERT_TRUE(holder->emplace(i, vec));
|
||||
}
|
||||
build_process(builder, holder);
|
||||
|
||||
// cleanup and rebuild
|
||||
ASSERT_EQ(0, builder->cleanup());
|
||||
}
|
||||
|
||||
#if defined(__GNUC__) || defined(__GNUG__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
@@ -3447,93 +3447,6 @@ TEST_F(HnswStreamerTest, TestGroupInBruteforceSearch) {
|
||||
}
|
||||
}
|
||||
|
||||
#if 0
|
||||
TEST_F(HnswStreamerTest, TestBinaryConverter) {
|
||||
uint32_t dimension = 2560;
|
||||
|
||||
IndexStreamer::Pointer streamer =
|
||||
IndexFactory::CreateStreamer("HnswStreamer");
|
||||
ASSERT_TRUE(streamer != nullptr);
|
||||
|
||||
ailego::Params params;
|
||||
// params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10);
|
||||
// params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 16);
|
||||
// params.set(PARAM_HNSW_STREAMER_EFCONSTRUCTION, 10);
|
||||
// params.set(PARAM_HNSW_STREAMER_EF, 5);
|
||||
params.set(PARAM_HNSW_STREAMER_BRUTE_FORCE_THRESHOLD, 1000U);
|
||||
|
||||
ailego::Params stg_params;
|
||||
|
||||
IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, dimension);
|
||||
index_meta_raw.set_metric("InnerProduct", 0, ailego::Params());
|
||||
|
||||
ailego::Params converter_params;
|
||||
auto converter = IndexFactory::CreateConverter("BinaryConverter");
|
||||
ASSERT_TRUE(converter != nullptr);
|
||||
|
||||
converter->init(index_meta_raw, converter_params);
|
||||
|
||||
IndexMeta index_meta = converter->meta();
|
||||
|
||||
auto reformer = IndexFactory::CreateReformer(index_meta.reformer_name());
|
||||
ASSERT_TRUE(reformer != nullptr);
|
||||
|
||||
ASSERT_EQ(0, reformer->init(index_meta.reformer_params()));
|
||||
|
||||
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
|
||||
ASSERT_EQ(0, storage->init(stg_params));
|
||||
ASSERT_EQ(0, storage->open(dir_ + "TestBinaryConverter.index", true));
|
||||
ASSERT_EQ(0, streamer->init(index_meta, params));
|
||||
ASSERT_EQ(0, streamer->open(storage));
|
||||
|
||||
size_t cnt = 5000U;
|
||||
auto ctx = streamer->create_context();
|
||||
ASSERT_TRUE(!!ctx);
|
||||
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dimension);
|
||||
|
||||
std::random_device rd;
|
||||
std::mt19937 gen(rd());
|
||||
|
||||
std::uniform_real_distribution<float> dist(-2.0, 2.0);
|
||||
std::vector<NumericalVector<float>> vecs;
|
||||
|
||||
for (size_t i = 0; i < cnt; i++) {
|
||||
NumericalVector<float> vec(dimension);
|
||||
for (size_t j = 0; j < dimension; ++j) {
|
||||
vec[j] = dist(gen);
|
||||
}
|
||||
|
||||
std::string new_vec;
|
||||
IndexQueryMeta new_meta;
|
||||
|
||||
ASSERT_EQ(0, reformer->convert(vec.data(), qmeta, &new_vec, &new_meta));
|
||||
ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx));
|
||||
|
||||
vecs.push_back(vec);
|
||||
}
|
||||
|
||||
size_t query_cnt = 200U;
|
||||
auto knnCtx = streamer->create_context();
|
||||
|
||||
float epison = 1e-6;
|
||||
for (size_t i = 0; i < query_cnt; i++) {
|
||||
auto &vec = vecs[i];
|
||||
std::string new_query;
|
||||
IndexQueryMeta new_meta;
|
||||
ASSERT_EQ(0, reformer->transform(vec.data(), qmeta, &new_query, &new_meta));
|
||||
|
||||
size_t topk = 50;
|
||||
knnCtx->set_topk(topk);
|
||||
ASSERT_EQ(0, streamer->search_impl(new_query.data(), new_meta, knnCtx));
|
||||
auto &results = knnCtx->result();
|
||||
ASSERT_EQ(topk, results.size());
|
||||
ASSERT_EQ(i, results[0].key());
|
||||
ASSERT_NEAR(0, results[0].score(), epison);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST_F(HnswStreamerTest, TestAddAndSearchWithID) {
|
||||
IndexStreamer::Pointer streamer =
|
||||
IndexFactory::CreateStreamer("HnswStreamer");
|
||||
@@ -3647,131 +3560,6 @@ TEST_F(HnswStreamerTest, TestAddAndSearchWithID) {
|
||||
// EXPECT_GT(cost, 2.0f);
|
||||
}
|
||||
|
||||
#if 0
|
||||
TEST_F(HnswStreamerTest, TestBasicRefiner) {
|
||||
uint32_t dimension = 1120;
|
||||
|
||||
IndexStreamer::Pointer base_streamer =
|
||||
IndexFactory::CreateStreamer("HnswStreamer");
|
||||
ASSERT_TRUE(base_streamer != nullptr);
|
||||
|
||||
IndexStreamer::Pointer refine_streamer =
|
||||
IndexFactory::CreateStreamer("FlatStreamer");
|
||||
ASSERT_TRUE(refine_streamer != nullptr);
|
||||
|
||||
IndexRefiner::Pointer refiner = IndexFactory::CreateRefiner("BasicRefiner");
|
||||
ASSERT_TRUE(refiner != nullptr);
|
||||
|
||||
ailego::Params params;
|
||||
IndexMeta index_meta(IndexMeta::DataType::DT_FP32, dimension);
|
||||
index_meta.set_metric("InnerProduct", 0, ailego::Params());
|
||||
|
||||
ailego::Params converter_params;
|
||||
auto converter = IndexFactory::CreateConverter("BinaryConverter");
|
||||
ASSERT_TRUE(converter != nullptr);
|
||||
|
||||
converter->init(index_meta, converter_params);
|
||||
|
||||
IndexMeta index_meta_binary = converter->meta();
|
||||
|
||||
auto reformer =
|
||||
IndexFactory::CreateReformer(index_meta_binary.reformer_name());
|
||||
ASSERT_TRUE(reformer != nullptr);
|
||||
|
||||
ASSERT_EQ(0, reformer->init(index_meta_binary.reformer_params()));
|
||||
|
||||
// base streamer
|
||||
ailego::Params base_stg_params;
|
||||
auto base_storage = IndexFactory::CreateStorage("MMapFileStorage");
|
||||
ASSERT_EQ(0, base_storage->init(base_stg_params));
|
||||
ASSERT_EQ(0, base_storage->open(dir_ + "TestBasicRefinerBase.index", true));
|
||||
ASSERT_EQ(0, base_streamer->init(index_meta_binary, params));
|
||||
ASSERT_EQ(0, base_streamer->open(base_storage));
|
||||
|
||||
auto base_ctx = base_streamer->create_context();
|
||||
ASSERT_TRUE(!!base_ctx);
|
||||
|
||||
// refine streamer
|
||||
ailego::Params refine_stg_params;
|
||||
auto refine_storage = IndexFactory::CreateStorage("MMapFileStorage");
|
||||
ASSERT_EQ(0, refine_storage->init(refine_stg_params));
|
||||
ASSERT_EQ(0,
|
||||
refine_storage->open(dir_ + "TestBasicRefinerRefine.index", true));
|
||||
ASSERT_EQ(0, refine_streamer->init(index_meta, params));
|
||||
ASSERT_EQ(0, refine_streamer->open(refine_storage));
|
||||
auto refine_ctx = refine_streamer->create_context();
|
||||
ASSERT_TRUE(!!refine_ctx);
|
||||
|
||||
ailego::Params refiner_params;
|
||||
ASSERT_EQ(0, refiner->init(base_streamer, refine_streamer, refiner_params));
|
||||
|
||||
auto ctx = refiner->create_context();
|
||||
ASSERT_TRUE(!!ctx);
|
||||
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dimension);
|
||||
|
||||
std::random_device rd;
|
||||
std::mt19937 gen(rd());
|
||||
|
||||
std::uniform_real_distribution<float> dist(-2.0, 2.0);
|
||||
std::vector<NumericalVector<float>> vecs;
|
||||
|
||||
size_t cnt = 5000U;
|
||||
for (size_t i = 0; i < cnt; i++) {
|
||||
NumericalVector<float> vec(dimension);
|
||||
for (size_t j = 0; j < dimension; ++j) {
|
||||
vec[j] = dist(gen);
|
||||
}
|
||||
|
||||
std::string binary_vec;
|
||||
IndexQueryMeta binary_qmeta;
|
||||
|
||||
ASSERT_EQ(0,
|
||||
reformer->convert(vec.data(), qmeta, &binary_vec, &binary_qmeta));
|
||||
ASSERT_EQ(0, refiner->add_impl(i, binary_vec.data(), binary_qmeta,
|
||||
vec.data(), qmeta, ctx));
|
||||
|
||||
vecs.push_back(vec);
|
||||
}
|
||||
|
||||
size_t query_cnt = 200U;
|
||||
// size_t query_cnt = 1U;
|
||||
|
||||
auto searcherCtx = refiner->create_context();
|
||||
|
||||
for (size_t i = 0; i < query_cnt; i++) {
|
||||
auto &vec = vecs[i];
|
||||
|
||||
// float abs_value{0};
|
||||
// for (size_t j = 0; j < dimension; ++j) {
|
||||
// std::cout << "dim: " << j << ", value: " << vec[j] << std::endl;
|
||||
|
||||
// abs_value += std::abs(vec[j]);
|
||||
// }
|
||||
// std::cout << "abs value: " << abs_value << std::endl;
|
||||
|
||||
std::string new_query;
|
||||
IndexQueryMeta binary_qmeta;
|
||||
ASSERT_EQ(
|
||||
0, reformer->transform(vec.data(), qmeta, &new_query, &binary_qmeta));
|
||||
|
||||
size_t topk = 50;
|
||||
searcherCtx->set_topk(topk);
|
||||
ASSERT_EQ(0, refiner->search_impl(new_query.data(), binary_qmeta,
|
||||
vec.data(), qmeta, searcherCtx));
|
||||
auto &results = searcherCtx->result();
|
||||
ASSERT_EQ(topk, results.size());
|
||||
ASSERT_EQ(i, results[0].key());
|
||||
|
||||
// for (size_t i = 0; i < results.size(); ++i) {
|
||||
// std::cout << i << ", id: " << results[i].index()
|
||||
// << ", score: " << results[i].score() << std::endl;
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
TEST_F(HnswStreamerTest, TestContiguousMemorySearch) {
|
||||
// Build index with mmap mode
|
||||
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
|
||||
|
||||
@@ -1405,258 +1405,6 @@ TEST_F(IVFSearcherTest, TestRowMajorInt8WithBuildMemory) {
|
||||
EXPECT_EQ(0, ret);
|
||||
}
|
||||
|
||||
TEST_F(IVFSearcherTest, TestColumnMajorBinaryWithBuildMemory) {
|
||||
IVFBuilder builder;
|
||||
dimension_ = 256;
|
||||
index_meta_.set_meta(IndexMeta::DataType::DT_BINARY32, dimension_);
|
||||
index_meta_.set_metric("Hamming", 0, Params());
|
||||
index_meta_.set_major_order(IndexMeta::MO_COLUMN);
|
||||
|
||||
int ret = builder.init(index_meta_, params_);
|
||||
EXPECT_EQ(0, ret);
|
||||
size_t fnum = 257;
|
||||
prepare_binary_index_holder(0, fnum);
|
||||
ret = builder.train(threads_, holder_);
|
||||
EXPECT_EQ(0, ret);
|
||||
ret = builder.build(threads_, holder_);
|
||||
EXPECT_EQ(0, ret);
|
||||
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("FileDumper");
|
||||
ret = dumper->create(index_path_);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
ret = builder.dump(dumper);
|
||||
EXPECT_EQ(0, dumper->close());
|
||||
EXPECT_EQ((size_t)fnum, builder.stats().built_count());
|
||||
EXPECT_EQ((size_t)fnum, builder.stats().dumped_count());
|
||||
EXPECT_EQ((size_t)0, builder.stats().discarded_count());
|
||||
|
||||
IVFSearcher searcher;
|
||||
Params params;
|
||||
params.set(PARAM_IVF_SEARCHER_SCAN_RATIO, 1.0);
|
||||
params.set(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD, 1);
|
||||
|
||||
ret = searcher.init(params);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
IndexStorage::Pointer container =
|
||||
IndexFactory::CreateStorage("MMapFileReadStorage");
|
||||
EXPECT_TRUE(!!container);
|
||||
|
||||
Params container_params;
|
||||
container_params.set("proxima.mmap_file.container.memory_warmup", true);
|
||||
container->init(container_params);
|
||||
ret = container->open(index_path_, false);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
ret = searcher.load(container, IndexMetric::Pointer());
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
BinaryVector<uint32_t> query(dimension_);
|
||||
for (size_t i = 0; i < dimension_; ++i) {
|
||||
query.set(i);
|
||||
}
|
||||
|
||||
size_t qnum = 63;
|
||||
BinaryVector<uint32_t> query1(dimension_ * qnum);
|
||||
for (size_t i = 0; i < qnum; ++i) {
|
||||
for (size_t j = 0; j < dimension_ && j < i; ++j) {
|
||||
query1.set(i * dimension_ + j);
|
||||
}
|
||||
}
|
||||
|
||||
auto context = searcher.create_context();
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_BINARY32, dimension_);
|
||||
|
||||
// single bf serch
|
||||
{
|
||||
size_t topk = 128;
|
||||
context->set_topk(topk);
|
||||
ret = searcher.search_bf_impl(query.data(), qmeta, context);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
const IndexDocumentList &result = context->result(0);
|
||||
EXPECT_EQ((size_t)topk, result.size());
|
||||
for (size_t i = 0; i < topk; ++i) {
|
||||
EXPECT_EQ((uint64_t)256 - i, result[i].key());
|
||||
EXPECT_FLOAT_EQ((float)i, result[i].score());
|
||||
}
|
||||
}
|
||||
|
||||
// batch bf serch
|
||||
{
|
||||
size_t topk = 1;
|
||||
context->set_topk(topk);
|
||||
ret = searcher.search_bf_impl(query1.data(), qmeta, qnum, context);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
for (size_t q = 0; q < qnum; ++q) {
|
||||
const IndexDocumentList &result = context->result(q);
|
||||
EXPECT_EQ((size_t)topk, result.size());
|
||||
EXPECT_EQ((uint64_t)q, result[0].key());
|
||||
EXPECT_FLOAT_EQ((float)0, result[0].score());
|
||||
}
|
||||
}
|
||||
|
||||
// single knn search
|
||||
{
|
||||
size_t topk = 100;
|
||||
context->set_topk(topk);
|
||||
ret = searcher.search_impl(query.data(), qmeta, context);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
const IndexDocumentList &result = context->result(0);
|
||||
EXPECT_EQ((size_t)topk, result.size());
|
||||
for (size_t i = 0; i < topk; ++i) {
|
||||
EXPECT_EQ((uint64_t)256 - i, result[i].key());
|
||||
EXPECT_FLOAT_EQ((float)i, result[i].score());
|
||||
}
|
||||
}
|
||||
|
||||
// batch knn serch
|
||||
{
|
||||
size_t topk = 1;
|
||||
context->set_topk(topk);
|
||||
ret = searcher.search_impl(query1.data(), qmeta, qnum, context);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
for (size_t q = 0; q < qnum; ++q) {
|
||||
const IndexDocumentList &result = context->result(q);
|
||||
EXPECT_EQ((size_t)topk, result.size());
|
||||
EXPECT_EQ((uint64_t)q, result[0].key());
|
||||
EXPECT_FLOAT_EQ((float)0, result[0].score());
|
||||
}
|
||||
}
|
||||
|
||||
ret = searcher.unload();
|
||||
EXPECT_EQ(0, ret);
|
||||
}
|
||||
|
||||
TEST_F(IVFSearcherTest, TestRowMajorBinaryWithBuildMemory) {
|
||||
IVFBuilder builder;
|
||||
dimension_ = 256;
|
||||
index_meta_.set_meta(IndexMeta::DataType::DT_BINARY32, dimension_);
|
||||
index_meta_.set_metric("Hamming", 0, Params());
|
||||
index_meta_.set_major_order(IndexMeta::MO_ROW);
|
||||
|
||||
int ret = builder.init(index_meta_, params_);
|
||||
EXPECT_EQ(0, ret);
|
||||
size_t fnum = 257;
|
||||
prepare_binary_index_holder(0, fnum);
|
||||
ret = builder.train(threads_, holder_);
|
||||
EXPECT_EQ(0, ret);
|
||||
ret = builder.build(threads_, holder_);
|
||||
EXPECT_EQ(0, ret);
|
||||
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("FileDumper");
|
||||
ret = dumper->create(index_path_);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
ret = builder.dump(dumper);
|
||||
EXPECT_EQ(0, dumper->close());
|
||||
EXPECT_EQ((size_t)fnum, builder.stats().built_count());
|
||||
EXPECT_EQ((size_t)fnum, builder.stats().dumped_count());
|
||||
EXPECT_EQ((size_t)0, builder.stats().discarded_count());
|
||||
|
||||
IVFSearcher searcher;
|
||||
Params params;
|
||||
params.set(PARAM_IVF_SEARCHER_SCAN_RATIO, 1.0);
|
||||
params.set(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD, 1);
|
||||
|
||||
ret = searcher.init(params);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
IndexStorage::Pointer container =
|
||||
IndexFactory::CreateStorage("MMapFileReadStorage");
|
||||
EXPECT_TRUE(!!container);
|
||||
|
||||
Params container_params;
|
||||
container_params.set("proxima.mmap_file.container.memory_warmup", true);
|
||||
container->init(container_params);
|
||||
ret = container->open(index_path_, false);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
ret = searcher.load(container, IndexMetric::Pointer());
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
BinaryVector<uint32_t> query(dimension_);
|
||||
for (size_t i = 0; i < dimension_; ++i) {
|
||||
query.set(i);
|
||||
}
|
||||
|
||||
size_t qnum = 63;
|
||||
BinaryVector<uint32_t> query1(dimension_ * qnum);
|
||||
for (size_t i = 0; i < qnum; ++i) {
|
||||
for (size_t j = 0; j < dimension_ && j < i; ++j) {
|
||||
query1.set(i * dimension_ + j);
|
||||
}
|
||||
}
|
||||
|
||||
auto context = searcher.create_context();
|
||||
IndexQueryMeta qmeta(IndexMeta::DataType::DT_BINARY32, dimension_);
|
||||
|
||||
// single bf serch
|
||||
{
|
||||
size_t topk = 128;
|
||||
context->set_topk(topk);
|
||||
ret = searcher.search_bf_impl(query.data(), qmeta, context);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
const IndexDocumentList &result = context->result(0);
|
||||
EXPECT_EQ((size_t)topk, result.size());
|
||||
for (size_t i = 0; i < topk; ++i) {
|
||||
EXPECT_EQ((uint64_t)256 - i, result[i].key());
|
||||
EXPECT_FLOAT_EQ((float)i, result[i].score());
|
||||
}
|
||||
}
|
||||
|
||||
// batch bf serch
|
||||
{
|
||||
size_t topk = 1;
|
||||
context->set_topk(topk);
|
||||
ret = searcher.search_bf_impl(query1.data(), qmeta, qnum, context);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
for (size_t q = 0; q < qnum; ++q) {
|
||||
const IndexDocumentList &result = context->result(q);
|
||||
EXPECT_EQ((size_t)topk, result.size());
|
||||
EXPECT_EQ((uint64_t)q, result[0].key());
|
||||
EXPECT_FLOAT_EQ((float)0, result[0].score());
|
||||
}
|
||||
}
|
||||
|
||||
// single knn search
|
||||
{
|
||||
size_t topk = 100;
|
||||
context->set_topk(topk);
|
||||
ret = searcher.search_impl(query.data(), qmeta, context);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
const IndexDocumentList &result = context->result(0);
|
||||
EXPECT_EQ((size_t)topk, result.size());
|
||||
for (size_t i = 0; i < topk; ++i) {
|
||||
EXPECT_EQ((uint64_t)256 - i, result[i].key());
|
||||
EXPECT_FLOAT_EQ((float)i, result[i].score());
|
||||
}
|
||||
}
|
||||
|
||||
// batch knn serch
|
||||
{
|
||||
size_t topk = 1;
|
||||
context->set_topk(topk);
|
||||
ret = searcher.search_impl(query1.data(), qmeta, qnum, context);
|
||||
EXPECT_EQ(0, ret);
|
||||
|
||||
for (size_t q = 0; q < qnum; ++q) {
|
||||
const IndexDocumentList &result = context->result(q);
|
||||
EXPECT_EQ((size_t)topk, result.size());
|
||||
EXPECT_EQ((uint64_t)q, result[0].key());
|
||||
EXPECT_FLOAT_EQ((float)0, result[0].score());
|
||||
}
|
||||
}
|
||||
|
||||
ret = searcher.unload();
|
||||
EXPECT_EQ(0, ret);
|
||||
}
|
||||
|
||||
TEST_F(IVFSearcherTest, TestSearchWithEmptyCentroid) {
|
||||
IVFBuilder builder;
|
||||
Params params;
|
||||
|
||||
Reference in New Issue
Block a user