Initial commit

This commit is contained in:
sanyi
2025-12-30 11:02:17 +08:00
commit 6524fabd80
841 changed files with 293919 additions and 0 deletions
+9
View File
@@ -0,0 +1,9 @@
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
include(${CMAKE_SOURCE_DIR}/cmake/option.cmake)
cc_directories(cluster)
cc_directories(flat)
cc_directories(flat_sparse)
cc_directories(ivf)
cc_directories(hnsw)
cc_directories(hnsw_sparse)
@@ -0,0 +1,14 @@
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
foreach(CC_SRCS ${ALL_TEST_SRCS})
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
cc_gtest(
NAME ${CC_TARGET}
STRICT
LIBS zvec_ailego core_framework core_utility core_metric core_knn_cluster
SRCS ${CC_SRCS}
INCS . ${CMAKE_SOURCE_DIR}/src/core ${CMAKE_SOURCE_DIR}/src/core/algorithm
)
endforeach()
@@ -0,0 +1,106 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include <random>
#include <ailego/container/params.h>
#include <gtest/gtest.h>
#include "framework/index_framework.h"
#include "framework/index_meta.h"
using namespace zvec::core;
using namespace zvec::ailego;
TEST(KmeansCluster, General) {
// Prepare index data
const uint32_t count = 5000u;
const uint32_t dimension = 33u;
IndexMeta index_meta;
index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
index_meta.set_metric("SquaredEuclidean", 0, zvec::ailego::Params());
std::shared_ptr<CompactIndexFeatures> features(
new CompactIndexFeatures(index_meta));
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(0.0, 5.0);
for (uint32_t i = 0; i < count; ++i) {
std::vector<float> vec(dimension);
for (size_t j = 0; j < dimension; ++j) {
vec[j] = dist(gen);
}
features->emplace(vec.data());
}
// Create a Kmeans cluster
// IndexCluster::Pointer cluster = std::make_shared<KmeansCluster>();
IndexCluster::Pointer cluster = IndexFactory::CreateCluster("KmeansCluster");
ASSERT_TRUE(!!cluster);
zvec::ailego::Params params;
params.set("proxima.general.cluster.count", 1);
params.set("proxima.kmeans.cluster.count", 56);
ASSERT_EQ(0, cluster->init(index_meta, params));
ASSERT_EQ(0, cluster->mount(features));
cluster->suggest(64u);
auto threads = std::make_shared<SingleQueueIndexThreads>();
std::cout << "---------- FIRST ----------\n";
std::vector<IndexCluster::Centroid> centroids;
std::vector<uint32_t> labels;
ASSERT_NE(0, cluster->classify(threads, centroids));
ASSERT_NE(0, cluster->label(threads, centroids, &labels));
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- SECOND ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- THIRD ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
ASSERT_EQ(0, cluster->classify(threads, centroids));
ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
}
@@ -0,0 +1,602 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include <random>
#include <ailego/algorithm/kmeans.h>
#include <ailego/container/params.h>
#include <gtest/gtest.h>
#include "framework/index_framework.h"
using namespace zvec::core;
using namespace zvec::ailego;
using namespace zvec::ailego;
TEST(OptKmeansCluster, General) {
// Prepare index data
const uint32_t count = 5000u;
const uint32_t dimension = 33u;
IndexMeta index_meta;
index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
std::shared_ptr<CompactIndexFeatures> features(
new CompactIndexFeatures(index_meta));
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(0.0, 5.0);
for (uint32_t i = 0; i < count; ++i) {
std::vector<float> vec(dimension);
for (size_t j = 0; j < dimension; ++j) {
vec[j] = dist(gen);
}
features->emplace(vec.data());
}
// Create a Kmeans cluster
IndexCluster::Pointer cluster =
IndexFactory::CreateCluster("OptKmeansCluster");
ASSERT_TRUE(!!cluster);
Params params;
params.set("proxima.general.cluster.count", 1);
params.set("proxima.optkmeans.cluster.count", 56);
ASSERT_EQ(0, cluster->init(index_meta, params));
ASSERT_EQ(0, cluster->mount(features));
cluster->suggest(64u);
auto threads = std::make_shared<SingleQueueIndexThreads>();
std::cout << "---------- FIRST ----------\n";
std::vector<IndexCluster::Centroid> centroids;
std::vector<uint32_t> labels;
ASSERT_NE(0, cluster->classify(threads, centroids));
ASSERT_NE(0, cluster->label(threads, centroids, &labels));
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- SECOND ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- THIRD ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
ASSERT_EQ(0, cluster->classify(threads, centroids));
ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
}
// TEST(OptKmeansCluster, NoEmptyCentroids) {
// // Prepare index data
// const uint32_t count = 500u;
// const uint32_t dimension = 8u;
// IndexMeta index_meta;
// index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
// index_meta.set_metric("SquaredEuclidean", 0, Params());
// std::shared_ptr<CompactIndexFeatures> features(
// new CompactIndexFeatures(index_meta));
// std::random_device rd;
// std::mt19937 gen(rd());
// std::uniform_real_distribution<float> dist(0.0, 5.0);
// for (uint32_t i = 0; i < count; ++i) {
// std::vector<float> vec(dimension);
// for (size_t j = 0; j < dimension; ++j) {
// vec[j] = dist(gen);
// }
// features->emplace(vec.data());
// }
// // Create a Kmeans cluster
// IndexCluster::Pointer cluster =
// IndexFactory::CreateCluster("OptKmeansCluster");
// ASSERT_TRUE(!!cluster);
// Params params;
// ASSERT_EQ(0, cluster->init(index_meta, params));
// ASSERT_EQ(0, cluster->mount(features));
// cluster->suggest(20u);
// auto threads = std::make_shared<SingleQueueIndexThreads>();
// std::vector<IndexCluster::Centroid> centroids;
// for (uint32_t i = 0; i < 3; ++i) {
// std::vector<float> vec(dimension);
// for (size_t j = 0; j < dimension; ++j) {
// vec[j] = NAN;
// }
// centroids.emplace_back(vec.data(), vec.size() * sizeof(float));
// }
// ASSERT_EQ(0, cluster->cluster(threads, centroids));
// ASSERT_EQ(3u, centroids.size());
// for (uint32_t i = 0; i < 3; ++i) {
// std::vector<float> vec(dimension);
// for (size_t j = 0; j < dimension; ++j) {
// vec[j] = dist(gen);
// }
// centroids.emplace_back(vec.data(), vec.size() * sizeof(float));
// }
// ASSERT_EQ(0, cluster->cluster(threads, centroids));
// ASSERT_EQ(6u, centroids.size());
// for (uint32_t i = 0; i < 3; ++i) {
// std::vector<float> vec(dimension);
// for (size_t j = 0; j < dimension; ++j) {
// vec[j] = NAN;
// }
// centroids.emplace_back(vec.data(), vec.size() * sizeof(float));
// }
// ASSERT_EQ(0, cluster->cluster(threads, centroids));
// ASSERT_EQ(9u, centroids.size());
// for (uint32_t i = 0; i < 3; ++i) {
// std::vector<float> vec(dimension);
// for (size_t j = 0; j < dimension; ++j) {
// vec[j] = dist(gen);
// }
// centroids.emplace_back(vec.data(), vec.size() * sizeof(float));
// }
// ASSERT_EQ(0, cluster->cluster(threads, centroids));
// ASSERT_EQ(12u, centroids.size());
// for (const auto &it : centroids) {
// const auto &vec = it.vector<float>();
// std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ",
// "
// << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() -
// 2]
// << ", " << vec[vec.size() - 1] << " }" << std::endl;
// }
// params.set("proxima.optkmeans.cluster.purge_empty", true);
// cluster->update(params);
// ASSERT_EQ(12u, centroids.size());
// ASSERT_EQ(0, cluster->cluster(threads, centroids));
// ASSERT_EQ(7u, centroids.size());
// for (const auto &it : centroids) {
// const auto &vec = it.vector<float>();
// std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ",
// "
// << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() -
// 2]
// << ", " << vec[vec.size() - 1] << " }" << std::endl;
// }
// }
TEST(OptKmeansCluster, BinaryGeneral) {
// Prepare index data
const uint32_t count = 5000u;
const uint32_t dimension = 1024u;
IndexMeta index_meta;
index_meta.set_meta(IndexMeta::DataType::DT_BINARY32, dimension);
index_meta.set_metric("SquaredEuclidean", 0, Params());
std::shared_ptr<CompactIndexFeatures> features(
new CompactIndexFeatures(index_meta));
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(0.0, 1.0);
for (uint32_t i = 0; i < count; ++i) {
BinaryVector<uint32_t> vec(dimension);
for (size_t j = 0; j < dimension; ++j) {
if (dist(gen) >= 0.5) {
vec.set(j);
}
}
features->emplace(vec.data());
}
std::cout << "---------- FIRST ----------\n";
// Create a Kmeans cluster
IndexCluster::Pointer cluster =
IndexFactory::CreateCluster("OptKmeansCluster");
ASSERT_TRUE(!!cluster);
Params params;
params.set("proxima.general.cluster.count", 1);
params.set("proxima.optkmeans.cluster.count", 56);
ASSERT_EQ(0, cluster->init(index_meta, params));
ASSERT_EQ(0, cluster->mount(features));
cluster->suggest(64u);
auto threads = std::make_shared<SingleQueueIndexThreads>();
std::cout << "---------- FIRST ----------\n";
std::vector<IndexCluster::Centroid> centroids;
std::vector<uint32_t> labels;
ASSERT_NE(0, cluster->classify(threads, centroids));
ASSERT_NE(0, cluster->label(threads, centroids, &labels));
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<uint32_t>();
uint mask = 0x1;
std::cout << it.follows() << " (" << it.score() << ") { "
<< !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", "
<< !!(vec[0] & (mask << 2)) << ", ... , "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 2))) << ", "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 1))) << " }"
<< std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- SECOND ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<uint32_t>();
uint mask = 0x1;
std::cout << it.follows() << " (" << it.score() << ") { "
<< !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", "
<< !!(vec[0] & (mask << 2)) << ", ... , "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 2))) << ", "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 1))) << " }"
<< std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- THIRD ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<uint32_t>();
uint mask = 0x1;
std::cout << it.follows() << " (" << it.score() << ") { "
<< !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", "
<< !!(vec[0] & (mask << 2)) << ", ... , "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 2))) << ", "
<< !!(vec[0] & (mask << !!(sizeof(uint32_t) - 1))) << " }"
<< std::endl;
ASSERT_EQ(0u, it.similars().size());
}
ASSERT_EQ(0, cluster->classify(threads, centroids));
ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
}
TEST(OptKmeansCluster, IN4General) {
// Prepare index data
const uint32_t count = 5000u;
const uint32_t dimension = 64u;
const uint32_t dimension_wrong = 66u;
IndexMeta index_meta;
index_meta.set_meta(IndexMeta::DataType::DT_INT4, dimension);
index_meta.set_metric("SquaredEuclidean", 0, Params());
IndexMeta index_meta_wrong;
index_meta_wrong.set_meta(IndexMeta::DataType::DT_INT4, dimension_wrong);
index_meta_wrong.set_metric("SquaredEuclidean", 0, Params());
std::shared_ptr<CompactIndexFeatures> features(
new CompactIndexFeatures(index_meta));
std::shared_ptr<CompactIndexFeatures> features_wrong(
new CompactIndexFeatures(index_meta_wrong));
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<uint8_t> dist(0, UINT8_MAX);
for (uint32_t i = 0; i < count; ++i) {
std::vector<uint8_t> vec(dimension / 2);
std::vector<uint8_t> vec_wrong(dimension_wrong / 2);
for (size_t j = 0; j < dimension / 2; ++j) {
vec[j] = dist(gen);
}
for (size_t j = 0; j < dimension_wrong / 2; ++j) {
vec_wrong[j] = dist(gen);
}
features->emplace(vec.data());
features_wrong->emplace(vec_wrong.data());
}
// Create a OptKmeans cluster
IndexCluster::Pointer cluster =
IndexFactory::CreateCluster("OptKmeansCluster");
ASSERT_TRUE(!!cluster);
Params params;
ASSERT_EQ(0, cluster->init(index_meta_wrong, params));
ASSERT_NE(0, cluster->mount(features_wrong));
params.set("proxima.general.cluster.count", 1);
params.set("proxima.optkmeans.cluster.count", 56);
ASSERT_EQ(0, cluster->init(index_meta, params));
ASSERT_EQ(0, cluster->mount(features));
cluster->suggest(64u);
auto threads = std::make_shared<SingleQueueIndexThreads>();
std::cout << "---------- FIRST ----------\n";
std::vector<IndexCluster::Centroid> centroids;
std::vector<uint32_t> labels;
ASSERT_NE(0, cluster->classify(threads, centroids));
ASSERT_NE(0, cluster->label(threads, centroids, &labels));
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- SECOND ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- THIRD ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
ASSERT_EQ(0, cluster->classify(threads, centroids));
ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
}
TEST(OptKmeansCluster, IN4Correctness) {
// Prepare index data
const uint32_t count = 5000u;
const uint32_t dimension = 64u;
IndexMeta index_meta1;
index_meta1.set_meta(IndexMeta::DataType::DT_INT8, dimension);
index_meta1.set_metric("SquaredEuclidean", 0, Params());
IndexMeta index_meta2;
index_meta2.set_meta(IndexMeta::DataType::DT_INT4, dimension);
index_meta2.set_metric("SquaredEuclidean", 0, Params());
std::shared_ptr<CompactIndexFeatures> features1(
new CompactIndexFeatures(index_meta1));
std::shared_ptr<CompactIndexFeatures> features2(
new CompactIndexFeatures(index_meta2));
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<int> dist(-8, 7);
// Generate features
for (size_t i = 0; i < count; ++i) {
NumericalVector<int8_t> vec1(dimension);
NibbleVector<int32_t> vec2(dimension);
for (size_t j = 0; j < dimension; ++j) {
int8_t val = (int8_t)dist(gen);
vec1[j] = val;
vec2.set(j, val);
}
features1->emplace(vec1.data());
features2->emplace(vec2.data());
}
// Create a OptKmeans cluster of int8, and cluster only once
IndexCluster::Pointer cluster_once =
IndexFactory::CreateCluster("OptKmeansCluster");
ASSERT_TRUE(!!cluster_once);
Params params_once;
params_once.set("proxima.general.cluster.count", 65);
params_once.set("proxima.optkmeans.cluster.count", 63);
params_once.set("proxima.optkmeans.cluster.max_iterations", 1);
// Use KMC2 to init centroids
params_once.set("proxima.optkmeans.cluster.markov_chain_length", 20);
ASSERT_EQ(0, cluster_once->init(index_meta1, params_once));
ASSERT_EQ(0, cluster_once->mount(features1));
cluster_once->suggest(63);
auto threads = std::make_shared<SingleQueueIndexThreads>();
// Cluster once and get centroids
std::vector<IndexCluster::Centroid> centroids1;
ASSERT_EQ(0, cluster_once->cluster(threads, centroids1));
// Use centroids_one as init centroids to both int8 and int4 cluster
// Create a int8 cluster
IndexCluster::Pointer cluster_int8 =
IndexFactory::CreateCluster("OptKmeansCluster");
ASSERT_TRUE(!!cluster_int8);
Params params_int8;
params_int8.set("proxima.general.cluster.count", 65);
params_int8.set("proxima.optkmeans.cluster.count", 63);
ASSERT_EQ(0, cluster_int8->init(index_meta1, params_int8));
ASSERT_EQ(0, cluster_int8->mount(features1));
cluster_int8->suggest(63u);
// Create a int4 cluster
IndexCluster::Pointer cluster_int4 =
IndexFactory::CreateCluster("OptKmeansCluster");
ASSERT_TRUE(!!cluster_int4);
Params params_int4;
params_int4.set("proxima.general.cluster.count", 65);
params_int4.set("proxima.optkmeans.cluster.count", 63);
ASSERT_EQ(0, cluster_int4->init(index_meta2, params_int4));
ASSERT_EQ(0, cluster_int4->mount(features2));
cluster_int4->suggest(63u);
std::vector<IndexCluster::Centroid> centroids2;
// Use centroids of int8 to init centroids of int4
for (size_t i = 0; i < centroids1.size(); ++i) {
NibbleVector<int8_t> nvec;
nvec.assign(reinterpret_cast<const int8_t *>(centroids1[i].feature()),
dimension);
IndexCluster::Centroid curr_centroid;
curr_centroid.set_score(centroids1[i].score());
curr_centroid.set_follows(centroids1[i].follows());
curr_centroid.set_feature(nvec.data(), nvec.dimension() >> 1);
centroids2.push_back(curr_centroid);
}
ASSERT_EQ(0, cluster_int8->cluster(threads, centroids1));
ASSERT_EQ(0, cluster_int4->cluster(threads, centroids2));
EXPECT_EQ(centroids1.size(), centroids2.size());
for (size_t i = 0; i < centroids1.size(); ++i) {
EXPECT_EQ(centroids1[i].follows(), centroids2[i].follows());
EXPECT_EQ(centroids1[i].score(), centroids2[i].score());
}
}
TEST(OptKmeansCluster, InnerProduct) {
// Prepare index data
const uint32_t count = 5000u;
const uint32_t dimension = 33u;
IndexMeta index_meta;
index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
index_meta.set_metric("InnerProduct", 0, Params());
std::shared_ptr<CompactIndexFeatures> features(
new CompactIndexFeatures(index_meta));
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(-1.0, 1.0);
for (uint32_t i = 0; i < count; ++i) {
std::vector<float> vec(dimension);
for (size_t j = 0; j < dimension; ++j) {
vec[j] = dist(gen);
}
features->emplace(vec.data());
}
// Create a Kmeans cluster
IndexCluster::Pointer cluster =
IndexFactory::CreateCluster("OptKmeansCluster");
ASSERT_TRUE(!!cluster);
Params params;
params.set("proxima.general.cluster.count", 1);
params.set("proxima.optkmeans.cluster.count", 56);
ASSERT_EQ(0, cluster->init(index_meta, params));
ASSERT_EQ(0, cluster->mount(features));
cluster->suggest(64u);
auto threads = std::make_shared<SingleQueueIndexThreads>();
std::cout << "---------- FIRST ----------\n";
std::vector<IndexCluster::Centroid> centroids;
std::vector<uint32_t> labels;
ASSERT_NE(0, cluster->classify(threads, centroids));
ASSERT_NE(0, cluster->label(threads, centroids, &labels));
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- SECOND ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
std::cout << "---------- THIRD ----------\n";
ASSERT_EQ(0, cluster->cluster(threads, centroids));
for (const auto &it : centroids) {
const auto &vec = it.vector<float>();
std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
<< vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
<< ", " << vec[vec.size() - 1] << " }" << std::endl;
ASSERT_EQ(0u, it.similars().size());
}
ASSERT_EQ(0, cluster->classify(threads, centroids));
ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
}
+14
View File
@@ -0,0 +1,14 @@
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
foreach(CC_SRCS ${ALL_TEST_SRCS})
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
cc_gtest(
NAME ${CC_TARGET}
STRICT
LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_flat
SRCS ${CC_SRCS}
INCS . ${CMAKE_SOURCE_DIR}/src/core ${CMAKE_SOURCE_DIR}/src/core/algorithm
)
endforeach()
@@ -0,0 +1,457 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "flat/flat_builder.h"
#include <future>
#include <iostream>
#include <vector>
#include <gtest/gtest.h>
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
using namespace zvec::core;
using namespace zvec::ailego;
using namespace std;
static inline size_t RandomDimension(void) {
std::mt19937 gen((std::random_device())());
return (std::uniform_int_distribution<size_t>(1, 129))(gen);
}
static size_t DIMENSION = RandomDimension();
class FlatBuilderTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
public:
static std::string dir_;
static IndexMeta meta_;
};
std::string FlatBuilderTest ::dir_("flat_builder_test");
IndexMeta FlatBuilderTest::meta_;
void FlatBuilderTest::SetUp(void) {
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_COLUMN);
}
//! self-check column-major and row-major search.
void FlatBuilderTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
void build_process(IndexBuilder::Pointer &builder,
IndexHolder::Pointer holder) {
Params params;
ASSERT_EQ(0, builder->init(FlatBuilderTest::meta_, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
std::string path = FlatBuilderTest::dir_ + "/TestGeneral";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(0UL, stats.discarded_count());
}
TEST_F(FlatBuilderTest, TestInitSuccess) {
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
}
TEST_F(FlatBuilderTest, TestInitFailedWithInvalidMeasure) {
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
meta_.set_metric("invalid", 0, Params());
Params params;
int ret = builder->init(meta_, params);
EXPECT_EQ(IndexError_InvalidArgument, ret);
}
TEST_F(FlatBuilderTest, TestInt8InvalidColumnMajor) {
size_t dim = (DIMENSION + 3) / 4 * 4;
meta_.set_meta(IndexMeta::DataType::DT_INT8, dim + 2);
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
ASSERT_EQ(IndexMeta::MO_COLUMN, meta_.major_order());
Params params;
ASSERT_NE(0, builder->init(meta_, params));
}
TEST_F(FlatBuilderTest, TestInt8WithRandomDimension) {
size_t dim = DIMENSION;
meta_.set_meta(IndexMeta::DataType::DT_INT8, dim);
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_UNDEFINED);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
}
TEST_F(FlatBuilderTest, TestBinaryInvalidColumnMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim + 2);
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "/TestGeneral";
}
TEST_F(FlatBuilderTest, TestBuildWithRowMajor) {
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_ROW);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_FP32>>(DIMENSION);
size_t doc_cnt = 2000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(DIMENSION);
for (size_t j = 0; j < DIMENSION; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
int ret = builder->train(holder);
EXPECT_EQ(0, ret);
ret = builder->build(holder);
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestInt8BuildWithRowMajor) {
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_meta(IndexMeta::DT_INT8, DIMENSION);
meta_.set_major_order(IndexMeta::MO_ROW);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_INT8>>(DIMENSION);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<int8_t> vec(DIMENSION);
for (size_t j = 0; j < DIMENSION; ++j) {
vec[j] = (int8_t)(i % 128);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
int ret = builder->train(holder);
EXPECT_EQ(0, ret);
ret = builder->build(holder);
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestBinaryBuildWithRowMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
meta_.set_major_order(IndexMeta::MO_ROW);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
BinaryVector<uint32_t> vec(dim);
for (size_t j = 0; j < dim && j < i; ++j) {
vec.set(j);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
int ret = builder->train(holder);
EXPECT_EQ(0, ret);
ret = builder->build(holder);
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestBuildWithColumnMajor) {
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_FP32>>(DIMENSION);
size_t doc_cnt = 2000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(DIMENSION);
for (size_t j = 0; j < DIMENSION; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
int ret = builder->train(holder);
EXPECT_EQ(0, ret);
ret = builder->build(holder);
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestInt8BuildWithColumnMajor) {
size_t dim = (DIMENSION + 3) / 4 * 4;
meta_.set_meta(IndexMeta::DataType::DT_INT8, dim);
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "/TestGeneral";
auto holder = std::make_shared<OnePassIndexHolder<IndexMeta::DT_INT8>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<int8_t> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = (int8_t)(i % 128);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
int ret = builder->train(holder);
EXPECT_EQ(0, ret);
ret = builder->build(holder);
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestBinaryBuildWithColumnMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
ASSERT_EQ(0, builder->init(meta_, params));
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
BinaryVector<uint32_t> vec(dim);
for (size_t j = 0; j < dim && j < i; ++j) {
vec.set(j);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
int ret = builder->train(holder);
EXPECT_EQ(0, ret);
ret = builder->build(holder);
EXPECT_EQ(0, ret);
}
TEST_F(FlatBuilderTest, TestWithRowMajor) {
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_ROW);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_FP32>>(DIMENSION);
size_t doc_cnt = 2000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(DIMENSION);
for (size_t j = 0; j < DIMENSION; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
build_process(builder, holder);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
}
TEST_F(FlatBuilderTest, TestInt8WithRowMajor) {
meta_.set_meta(IndexMeta::DataType::DT_INT8, DIMENSION);
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_ROW);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_INT8>>(DIMENSION);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<int8_t> vec(DIMENSION);
for (size_t j = 0; j < DIMENSION; ++j) {
vec[j] = (int8_t)(i % 128);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
build_process(builder, holder);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
}
TEST_F(FlatBuilderTest, TestBinaryWithRowMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
meta_.set_major_order(IndexMeta::MO_ROW);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
BinaryVector<uint32_t> vec(dim);
for (size_t j = 0; j < dim && j < i; ++j) {
vec.set(j);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
build_process(builder, holder);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
}
TEST_F(FlatBuilderTest, TestWithColumnMajor) {
meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_FP32>>(DIMENSION);
size_t doc_cnt = 2000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(DIMENSION);
for (size_t j = 0; j < DIMENSION; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
build_process(builder, holder);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
}
TEST_F(FlatBuilderTest, TestInt8WithColumnMajor) {
size_t dim = (DIMENSION + 3) / 4 * 4;
meta_.set_meta(IndexMeta::DataType::DT_INT8, dim);
meta_.set_metric("SquaredEuclidean", 0, Params());
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
std::string path = dir_ + "/TestGeneral";
auto holder = std::make_shared<OnePassIndexHolder<IndexMeta::DT_INT8>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<int8_t> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = (int8_t)(i % 128);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
build_process(builder, holder);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
}
TEST_F(FlatBuilderTest, TestBinaryWithColumnMajor) {
size_t dim = (DIMENSION + 31) / 32 * 32;
meta_.set_metric("Hamming", 0, Params());
meta_.set_meta(IndexMeta::DT_BINARY32, dim);
meta_.set_major_order(IndexMeta::MO_COLUMN);
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
ASSERT_NE(builder, nullptr);
Params params;
std::string path = dir_ + "/TestGeneral";
auto holder =
std::make_shared<OnePassIndexHolder<IndexMeta::DT_BINARY32>>(dim);
size_t doc_cnt = 128UL;
for (size_t i = 0; i < doc_cnt; i++) {
BinaryVector<uint32_t> vec(dim);
for (size_t j = 0; j < dim && j < i; ++j) {
vec.set(j);
}
ASSERT_TRUE(holder->emplace(i, vec));
}
build_process(builder, holder);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
}
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,362 @@
#include <future>
#include <string>
#include <vector>
#include <ailego/buffer/buffer_manager.h>
#include <ailego/utility/math_helper.h>
#include <ailego/utility/memory_helper.h>
#include <framework/index_framework.h>
#include <framework/index_streamer.h>
#include <gtest/gtest.h>
using namespace zvec::core;
using namespace zvec::ailego;
using namespace std;
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
constexpr size_t static dim = 16;
class FlatStreamerTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
void hybrid_scale(std::vector<float> &dense_value,
std::vector<float> &sparse_value, float alpha_scale);
static std::string dir_;
static std::shared_ptr<IndexMeta> index_meta_ptr_;
};
std::string FlatStreamerTest::dir_("streamer_test/");
std::shared_ptr<IndexMeta> FlatStreamerTest::index_meta_ptr_;
void FlatStreamerTest::SetUp(void) {
index_meta_ptr_.reset(new (std::nothrow)
IndexMeta(IndexMeta::DataType::DT_FP32, dim));
index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params());
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
void FlatStreamerTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
TEST_F(FlatStreamerTest, TestLinearSearch) {
BufferManager::Instance().init(300 * 1024 / 2 * 1024, 1);
IndexStreamer::Pointer write_streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_TRUE(write_streamer != nullptr);
Params params;
ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearch", true));
ASSERT_EQ(0, write_streamer->open(storage));
auto ctx = write_streamer->create_context();
ASSERT_TRUE(!!ctx);
size_t cnt = 10000UL;
IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
for (size_t i = 0; i < cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
write_streamer->add_impl(i, vec.data(), qmeta, ctx);
}
write_streamer->flush(0UL);
write_streamer->close();
write_streamer.reset();
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
auto read_storage = IndexFactory::CreateStorage("BufferStorage");
ASSERT_NE(nullptr, read_storage);
ASSERT_EQ(0, read_storage->init(stg_params));
ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearch", false));
ASSERT_EQ(0, read_streamer->open(read_storage));
size_t topk = 3;
auto provider = read_streamer->create_provider();
for (size_t i = 0; i < cnt; i += 1) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result1 = ctx->result();
ASSERT_EQ(topk, result1.size());
IndexStorage::MemoryBlock block;
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
const float *data = (float *)block.data();
for (size_t j = 0; j < dim; ++j) {
ASSERT_EQ(data[j], i);
}
ASSERT_EQ(i, result1[0].key());
for (size_t j = 0; j < dim; ++j) {
vec[j] = i + 0.1f;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result2 = ctx->result();
ASSERT_EQ(topk, result2.size());
ASSERT_EQ(i, result2[0].key());
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
}
ctx->set_topk(100U);
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = 10.1f;
}
ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
auto &result = ctx->result();
ASSERT_EQ(100U, result.size());
ASSERT_EQ(10, result[0].key());
ASSERT_EQ(11, result[1].key());
ASSERT_EQ(5, result[10].key());
ASSERT_EQ(0, result[20].key());
ASSERT_EQ(30, result[30].key());
ASSERT_EQ(35, result[35].key());
ASSERT_EQ(99, result[99].key());
ElapsedTime elapsed_time;
for (size_t i = 0; i < cnt; i += 1) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result1 = ctx->result();
ASSERT_EQ(topk, result1.size());
IndexStorage::MemoryBlock block;
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
const float *data = (float *)block.data();
for (size_t j = 0; j < dim; ++j) {
ASSERT_EQ(data[j], i);
}
ASSERT_EQ(i, result1[0].key());
for (size_t j = 0; j < dim; ++j) {
vec[j] = i + 0.1f;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result2 = ctx->result();
ASSERT_EQ(topk, result2.size());
ASSERT_EQ(i, result2[0].key());
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
}
read_streamer->close();
read_streamer.reset();
cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
}
TEST_F(FlatStreamerTest, TestLinearSearchMMap) {
BufferManager::Instance().init(3 * 1024 / 2 * 1024, 1);
IndexStreamer::Pointer write_streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_TRUE(write_streamer != nullptr);
Params params;
ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearchMMap", true));
ASSERT_EQ(0, write_streamer->open(storage));
auto ctx = write_streamer->create_context();
ASSERT_TRUE(!!ctx);
size_t cnt = 10000UL;
IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
for (size_t i = 0; i < cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
write_streamer->add_impl(i, vec.data(), qmeta, ctx);
}
write_streamer->flush(0UL);
write_streamer->close();
write_streamer.reset();
ElapsedTime elapsed_time;
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, read_storage);
ASSERT_EQ(0, read_storage->init(stg_params));
ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearchMMap", false));
ASSERT_EQ(0, read_streamer->open(read_storage));
size_t topk = 3;
auto provider = read_streamer->create_provider();
for (size_t i = 0; i < cnt; i += 1) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result1 = ctx->result();
ASSERT_EQ(topk, result1.size());
IndexStorage::MemoryBlock block;
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
const float *data = (float *)block.data();
for (size_t j = 0; j < dim; ++j) {
ASSERT_EQ(data[j], i);
}
ASSERT_EQ(i, result1[0].key());
for (size_t j = 0; j < dim; ++j) {
vec[j] = i + 0.1f;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result2 = ctx->result();
ASSERT_EQ(topk, result2.size());
ASSERT_EQ(i, result2[0].key());
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
}
ctx->set_topk(100U);
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = 10.1f;
}
ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
auto &result = ctx->result();
ASSERT_EQ(100U, result.size());
ASSERT_EQ(10, result[0].key());
ASSERT_EQ(11, result[1].key());
ASSERT_EQ(5, result[10].key());
ASSERT_EQ(0, result[20].key());
ASSERT_EQ(30, result[30].key());
ASSERT_EQ(35, result[35].key());
ASSERT_EQ(99, result[99].key());
read_streamer->close();
read_streamer.reset();
cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
}
TEST_F(FlatStreamerTest, TestBufferStorage) {
BufferManager::Instance().init(10 * 1024 * 1024, 1);
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_TRUE(streamer != nullptr);
const int dim = 16;
IndexMeta meta = IndexMeta(IndexMeta::DT_FP32, dim);
meta.set_metric("SquaredEuclidean", 0, Params());
Params params;
EXPECT_EQ(0, streamer->init(meta, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
Params stg_params;
EXPECT_EQ(0, storage->init(stg_params));
EXPECT_EQ(0, storage->open(dir_ + "/Test/LinearSearch", true));
EXPECT_EQ(0, streamer->open(storage));
auto ctx = streamer->create_context();
ASSERT_TRUE(!!ctx);
size_t cnt = 1000UL;
IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
for (size_t i = 0; i < cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
streamer->add_impl(i, vec.data(), qmeta, ctx);
}
streamer->flush(0UL);
streamer.reset();
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_TRUE(read_streamer != nullptr);
EXPECT_EQ(0, read_streamer->init(meta, params));
auto read_storage = IndexFactory::CreateStorage("BufferStorage");
ASSERT_NE(nullptr, read_storage);
EXPECT_EQ(0, read_storage->init(stg_params));
EXPECT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearch", false));
EXPECT_EQ(0, read_streamer->open(read_storage));
auto read_ctx = read_streamer->create_context();
auto provider = read_streamer->create_provider();
size_t topk = 3;
for (size_t i = 0; i < cnt; i += 1) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
read_ctx->set_topk(topk);
EXPECT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, read_ctx));
auto &result1 = read_ctx->result();
EXPECT_EQ(topk, result1.size());
for (size_t j = 0; j < dim; ++j) {
const float *data = (float *)provider->get_vector(result1[0].key());
EXPECT_EQ(data[j], i);
}
EXPECT_EQ(i, result1[0].key());
for (size_t j = 0; j < dim; ++j) {
vec[j] = i + 0.1f;
}
read_ctx->set_topk(topk);
EXPECT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, read_ctx));
auto &result2 = read_ctx->result();
EXPECT_EQ(topk, result2.size());
EXPECT_EQ(i, result2[0].key());
EXPECT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
EXPECT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
}
read_ctx->set_topk(100U);
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = 10.1f;
}
EXPECT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, read_ctx));
auto &result = read_ctx->result();
EXPECT_EQ(100U, result.size());
EXPECT_EQ(10, result[0].key());
EXPECT_EQ(11, result[1].key());
EXPECT_EQ(5, result[10].key());
EXPECT_EQ(0, result[20].key());
EXPECT_EQ(30, result[30].key());
EXPECT_EQ(35, result[35].key());
EXPECT_EQ(99, result[99].key());
read_streamer->flush(0UL);
read_streamer.reset();
}
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
@@ -0,0 +1,140 @@
#include <future>
#include <string>
#include <vector>
#include <ailego/buffer/buffer_manager.h>
#include <ailego/utility/math_helper.h>
#include <ailego/utility/memory_helper.h>
#include <framework/index_framework.h>
#include <framework/index_streamer.h>
#include <gtest/gtest.h>
using namespace zvec::core;
using namespace zvec::ailego;
using namespace std;
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
constexpr size_t static dim = 128;
class FlatStreamerTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
void hybrid_scale(std::vector<float> &dense_value,
std::vector<float> &sparse_value, float alpha_scale);
static std::string dir_;
static std::shared_ptr<IndexMeta> index_meta_ptr_;
};
std::string FlatStreamerTest::dir_("streamer_test/");
std::shared_ptr<IndexMeta> FlatStreamerTest::index_meta_ptr_;
void FlatStreamerTest::SetUp(void) {
index_meta_ptr_.reset(new (std::nothrow)
IndexMeta(IndexMeta::DataType::DT_FP32, dim));
index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params());
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
void FlatStreamerTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
TEST_F(FlatStreamerTest, TestLinearSearchMMap) {
BufferManager::Instance().init(50 * 1024 * 1024, 1);
IndexStreamer::Pointer write_streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_TRUE(write_streamer != nullptr);
Params params;
ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearchMMap", true));
ASSERT_EQ(0, write_streamer->open(storage));
auto ctx = write_streamer->create_context();
ASSERT_TRUE(!!ctx);
size_t data_cnt = 300000UL, cnt = 500UL;
IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
for (size_t i = 0; i < data_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
write_streamer->add_impl(i, vec.data(), qmeta, ctx);
}
write_streamer->flush(0UL);
write_streamer->close();
write_streamer.reset();
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("FlatStreamer");
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
auto read_storage = IndexFactory::CreateStorage("BufferStorage");
ASSERT_NE(nullptr, read_storage);
ASSERT_EQ(0, read_storage->init(stg_params));
ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearchMMap", false));
ASSERT_EQ(0, read_streamer->open(read_storage));
size_t topk = 30;
ElapsedTime elapsed_time;
for (size_t i = 0; i < cnt; i += 1) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
// auto &result1 = ctx->result();
// ASSERT_EQ(topk, result1.size());
// ASSERT_EQ(i, result1[0].key());
// for (size_t j = 0; j < dim; ++j) {
// vec[j] = i + 0.1f;
// }
// ctx->set_topk(topk);
// ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
// auto &result2 = ctx->result();
// ASSERT_EQ(topk, result2.size());
// ASSERT_EQ(i, result2[0].key());
// ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
// ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
}
cout << "Elapsed time: " << elapsed_time.micro_seconds() << " us" << endl;
// ctx->set_topk(100U);
// NumericalVector<float> vec(dim);
// for (size_t j = 0; j < dim; ++j) {
// vec[j] = 10.1f;
// }
// ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
// auto &result = ctx->result();
// ASSERT_EQ(100U, result.size());
// ASSERT_EQ(10, result[0].key());
// ASSERT_EQ(11, result[1].key());
// ASSERT_EQ(5, result[10].key());
// ASSERT_EQ(0, result[20].key());
// ASSERT_EQ(30, result[30].key());
// ASSERT_EQ(35, result[35].key());
// ASSERT_EQ(99, result[99].key());
read_streamer->close();
read_streamer.reset();
// cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
}
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,14 @@
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
foreach(CC_SRCS ${ALL_TEST_SRCS})
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
cc_gtest(
NAME ${CC_TARGET}
STRICT
LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_flat_sparse
SRCS ${CC_SRCS}
INCS . ${CMAKE_SOURCE_DIR}/src/core ${CMAKE_SOURCE_DIR}/src/core/algorithm
)
endforeach()
@@ -0,0 +1,301 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "flat_sparse/flat_sparse_builder.h"
#include <future>
#include <iostream>
#include <vector>
#include <gtest/gtest.h>
using namespace zvec::core;
using namespace zvec::ailego;
using namespace std;
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
class FlatSparseBuilderTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
static std::string _dir;
static shared_ptr<IndexMeta> _index_meta_ptr;
};
std::string FlatSparseBuilderTest::_dir("FlatSparseBuilderTest");
shared_ptr<IndexMeta> FlatSparseBuilderTest::_index_meta_ptr;
void FlatSparseBuilderTest::SetUp(void) {
_index_meta_ptr.reset(new (nothrow) IndexMeta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32));
_index_meta_ptr->set_metric("InnerProductSparse", 0, Params());
}
void FlatSparseBuilderTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str());
system(cmdBuf);
}
TEST_F(FlatSparseBuilderTest, TestGeneral) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("FlatSparseBuilder");
ASSERT_NE(builder, nullptr);
auto holder = make_shared<OnePassIndexSparseHolder<IndexMeta::DT_FP32>>();
uint32_t sparse_count = 4;
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder->emplace(i, vec));
}
Params params;
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestGeneral";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_EQ(stats.built_costtime(), 0UL);
ASSERT_GT(stats.dumped_costtime(), 0UL);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
auto holder2 = make_shared<MultiPassIndexSparseHolder<IndexMeta::DT_FP32>>();
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder2->emplace(i, vec));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_EQ(stats.built_costtime(), 0UL);
}
TEST_F(FlatSparseBuilderTest, TestIndexThreads) {
IndexBuilder::Pointer builder1 =
IndexFactory::CreateBuilder("FlatSparseBuilder");
ASSERT_NE(builder1, nullptr);
IndexBuilder::Pointer builder2 =
IndexFactory::CreateBuilder("FlatSparseBuilder");
ASSERT_NE(builder2, nullptr);
auto holder = make_shared<MultiPassIndexSparseHolder<IndexMeta::DT_FP32>>();
size_t doc_cnt = 1000UL;
uint32_t sparse_count = 32;
for (size_t i = 0; i < doc_cnt; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder->emplace(i, vec));
}
Params params;
std::srand(Realtime::MilliSeconds());
auto threads =
std::make_shared<SingleQueueIndexThreads>(std::rand() % 4, false);
ASSERT_EQ(0, builder1->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder2->init(*_index_meta_ptr, params));
auto build_index1 = [&]() {
ASSERT_EQ(0, builder1->train(threads, holder));
ASSERT_EQ(0, builder1->build(threads, holder));
};
auto build_index2 = [&]() {
ASSERT_EQ(0, builder2->train(threads, holder));
ASSERT_EQ(0, builder2->build(threads, holder));
};
auto t1 = std::async(std::launch::async, build_index1);
auto t2 = std::async(std::launch::async, build_index2);
t1.wait();
t2.wait();
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestIndexThreads";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder1->dump(dumper));
ASSERT_EQ(0, dumper->close());
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder2->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats1 = builder1->stats();
ASSERT_EQ(doc_cnt, stats1.built_count());
auto &stats2 = builder2->stats();
ASSERT_EQ(doc_cnt, stats2.built_count());
}
TEST_F(FlatSparseBuilderTest, TestHalfFloatConverter) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("FlatSparseBuilder");
ASSERT_NE(builder, nullptr);
auto holder = make_shared<OnePassIndexSparseHolder<IndexMeta::DT_FP32>>();
uint32_t sparse_count = 4;
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder->emplace(i, vec));
}
Params converter_params;
auto converter = IndexFactory::CreateConverter("HalfFloatSparseConverter");
converter->init(*_index_meta_ptr, converter_params);
IndexMeta index_meta = converter->meta();
converter->transform(holder);
auto converted_holder = converter->sparse_result();
Params params;
ASSERT_EQ(0, builder->init(index_meta, converter_params));
ASSERT_EQ(0, builder->train(converted_holder));
ASSERT_EQ(0, builder->build(converted_holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestHalFloatConverter";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_EQ(stats.built_costtime(), 0UL);
ASSERT_GT(stats.dumped_costtime(), 0UL);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
auto holder2 = make_shared<MultiPassIndexSparseHolder<IndexMeta::DT_FP32>>();
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder2->emplace(i, vec));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_EQ(stats.built_costtime(), 0UL);
}
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
@@ -0,0 +1,822 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <future>
#include <iostream>
#include <random>
#include <vector>
#include <ailego/container/vector.h>
#include <ailego/math/norm2_matrix.h>
#include <ailego/utility/math_helper.h>
#include <gtest/gtest.h>
#include "framework/index_factory.h"
#include "framework/index_meta.h"
using namespace zvec::core;
using namespace zvec::ailego;
using namespace std;
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
constexpr size_t static sparse_dim_count = 16;
class FlatSparseSearcherTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
void generate_sparse_data(
size_t cnt, uint32_t sparse_dim_count,
std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
std::vector<NumericalVector<float>> &sparse_vec_list, bool norm);
static std::string dir_;
static std::shared_ptr<IndexMeta> index_meta_ptr_;
};
std::string FlatSparseSearcherTest::dir_("searcher_test/");
std::shared_ptr<IndexMeta> FlatSparseSearcherTest::index_meta_ptr_;
void FlatSparseSearcherTest::generate_sparse_data(
size_t cnt, uint32_t sparse_dim_count,
std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
std::vector<NumericalVector<float>> &sparse_vec_list, bool norm) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(-1.0, 1.0);
for (size_t i = 0; i < cnt; ++i) {
// prepare sparse
NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
NumericalVector<float> sparse_vec(sparse_dim_count);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_vec[j] = dist(gen);
}
float norm;
Norm2Matrix<float, 1>::Compute(sparse_vec.data(), sparse_dim_count, &norm);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_vec[j] = sparse_vec[j] / norm;
}
sparse_indices_list.push_back(sparse_indices);
sparse_vec_list.push_back(sparse_vec);
}
}
void FlatSparseSearcherTest::SetUp(void) {
IndexLoggerBroker::SetLevel(2);
index_meta_ptr_.reset(new IndexMeta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32));
index_meta_ptr_->set_metric("InnerProductSparse", 0, Params());
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
void FlatSparseSearcherTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
TEST_F(FlatSparseSearcherTest, TestGeneral) {
// init storage
Params stg_params;
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_TRUE(storage != nullptr);
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true));
// init streamer
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("FlatSparseStreamer");
ASSERT_TRUE(streamer != nullptr);
IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32);
index_meta.set_metric("InnerProductSparse", 0, Params());
Params params;
ASSERT_EQ(0, streamer->init(index_meta, params));
ASSERT_EQ(0, streamer->open(storage));
// generate sparse data
size_t sparse_dim_count = 32;
size_t cnt = 100U;
std::vector<NumericalVector<uint32_t>> sparse_indices_list;
std::vector<NumericalVector<float>> sparse_vec_list;
generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
sparse_vec_list, true);
// test add data
auto ctx = streamer->create_context();
ASSERT_TRUE(!!ctx);
IndexQueryMeta qmeta(IndexMeta::DT_FP32);
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count,
sparse_indices_list[i].data(),
sparse_vec_list[i].data(), qmeta, ctx));
}
// test get data
uint32_t sparse_count;
std::string sparse_indices_buffer;
std::string sparse_values_buffer;
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(
0, streamer->get_sparse_vector(i, &sparse_count, &sparse_indices_buffer,
&sparse_values_buffer));
ASSERT_EQ(sparse_dim_count, sparse_count);
const uint32_t *sparse_indices_ptr =
reinterpret_cast<const uint32_t *>(sparse_indices_buffer.data());
const float *sparse_values_ptr =
reinterpret_cast<const float *>(sparse_values_buffer.data());
for (size_t j = 0; j < sparse_count; ++j) {
ASSERT_EQ(sparse_indices_ptr[j], sparse_indices_list[i][j]);
ASSERT_FLOAT_EQ(sparse_values_ptr[j], sparse_vec_list[i][j]);
// std::cout << "1: " << sparse_values_ptr[j]
// << " 2: " << sparse_vec_list[i][j] << std::endl;
}
// must clear ^_^
sparse_indices_buffer.clear();
sparse_values_buffer.clear();
}
// test dump
auto path = dir_ + "/TestGeneral_dump";
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, streamer->dump(dumper));
ASSERT_EQ(0, streamer->close());
ASSERT_EQ(0, dumper->close());
// do searcher get vector
IndexSearcher::Pointer searcher =
IndexFactory::CreateSearcher("FlatSparseSearcher");
auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_TRUE(read_storage != nullptr);
ASSERT_TRUE(searcher != nullptr);
ASSERT_EQ(0, read_storage->open(path, false));
ASSERT_EQ(0, searcher->init(Params()));
ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
// test searcher get data
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(
0, searcher->get_sparse_vector(i, &sparse_count, &sparse_indices_buffer,
&sparse_values_buffer));
ASSERT_EQ(sparse_dim_count, sparse_count);
const uint32_t *sparse_indices_ptr =
reinterpret_cast<const uint32_t *>(sparse_indices_buffer.data());
const float *sparse_values_ptr =
reinterpret_cast<const float *>(sparse_values_buffer.data());
for (size_t j = 0; j < sparse_count; ++j) {
ASSERT_EQ(sparse_indices_ptr[j], sparse_indices_list[i][j]);
ASSERT_FLOAT_EQ(sparse_values_ptr[j], sparse_vec_list[i][j]);
// std::cout << "1: " << sparse_values_ptr[j]
// << " 2: " << sparse_vec_list[i][j] << std::endl;
}
// must clear ^_^
sparse_indices_buffer.clear();
sparse_values_buffer.clear();
}
}
TEST_F(FlatSparseSearcherTest, TestStreamerDump) {
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("FlatSparseStreamer");
ASSERT_NE(streamer, nullptr);
Params params;
Params stg_params;
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestStreamerDump.index", true));
ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
ASSERT_EQ(0, streamer->open(storage));
size_t cnt = 10000U;
auto ctx = streamer->create_context();
ASSERT_TRUE(!!ctx);
std::vector<NumericalVector<uint32_t>> sparse_indices_list;
std::vector<NumericalVector<float>> sparse_vec_list;
generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
sparse_vec_list, true);
IndexQueryMeta qmeta(IndexMeta::DT_FP32);
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count,
sparse_indices_list[i].data(),
sparse_vec_list[i].data(), qmeta, ctx));
}
auto path = dir_ + "/TestStreamerDump";
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, streamer->dump(dumper));
ASSERT_EQ(0, streamer->close());
ASSERT_EQ(0, dumper->close());
// do searcher knn
IndexSearcher::Pointer searcher =
IndexFactory::CreateSearcher("FlatSparseSearcher");
auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_EQ(0, read_storage->open(path, false));
ASSERT_TRUE(searcher != nullptr);
ASSERT_EQ(0, searcher->init(Params()));
ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
auto linearCtx = searcher->create_context();
auto knnCtx = searcher->create_context();
size_t topk = 200;
linearCtx->set_topk(topk);
knnCtx->set_topk(topk);
uint64_t knnTotalTime = 0;
uint64_t linearTotalTime = 0;
for (size_t i = 0; i < cnt; i += 50) {
const auto &sparse_indices = sparse_indices_list[i];
const auto &sparse_vec = sparse_vec_list[i];
auto t1 = Realtime::MicroSeconds();
ASSERT_EQ(0, searcher->search_impl(sparse_dim_count, sparse_indices.data(),
sparse_vec.data(), qmeta, knnCtx));
auto t2 = Realtime::MicroSeconds();
ASSERT_EQ(0,
searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
sparse_vec.data(), qmeta, linearCtx));
auto t3 = Realtime::MicroSeconds();
knnTotalTime += t2 - t1;
linearTotalTime += t3 - t2;
auto &knnResult = knnCtx->result();
auto &linearResult = linearCtx->result();
ASSERT_EQ(topk, linearResult.size());
ASSERT_EQ(topk, knnResult.size());
ASSERT_EQ(i, linearResult[0].key());
for (size_t k = 0; k < topk; ++k) {
ASSERT_EQ(linearResult[k].key(), knnResult[k].key());
}
}
printf("linear: %zu, knn: %zu\n", (size_t)linearTotalTime,
(size_t)knnTotalTime);
}
TEST_F(FlatSparseSearcherTest, TestLoadClose) {
Params stg_params;
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_TRUE(storage != nullptr);
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true));
// init streamer
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("FlatSparseStreamer");
ASSERT_TRUE(streamer != nullptr);
IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32);
index_meta.set_metric("InnerProductSparse", 0, Params());
Params params;
ASSERT_EQ(0, streamer->init(index_meta, params));
ASSERT_EQ(0, streamer->open(storage));
// generate sparse data
size_t sparse_dim_count = 32;
size_t cnt = 100U;
std::vector<NumericalVector<uint32_t>> sparse_indices_list;
std::vector<NumericalVector<float>> sparse_vec_list;
generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
sparse_vec_list, true);
// test add data
auto ctx = streamer->create_context();
ASSERT_TRUE(!!ctx);
IndexQueryMeta qmeta(IndexMeta::DT_FP32);
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count,
sparse_indices_list[i].data(),
sparse_vec_list[i].data(), qmeta, ctx));
}
// test get data
uint32_t sparse_count;
std::string sparse_indices_buffer;
std::string sparse_values_buffer;
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(
0, streamer->get_sparse_vector(i, &sparse_count, &sparse_indices_buffer,
&sparse_values_buffer));
ASSERT_EQ(sparse_dim_count, sparse_count);
const uint32_t *sparse_indices_ptr =
reinterpret_cast<const uint32_t *>(sparse_indices_buffer.data());
const float *sparse_values_ptr =
reinterpret_cast<const float *>(sparse_values_buffer.data());
for (size_t j = 0; j < sparse_count; ++j) {
ASSERT_EQ(sparse_indices_ptr[j], sparse_indices_list[i][j]);
ASSERT_FLOAT_EQ(sparse_values_ptr[j], sparse_vec_list[i][j]);
// std::cout << "1: " << sparse_values_ptr[j]
// << " 2: " << sparse_vec_list[i][j] << std::endl;
}
// must clear ^_^
sparse_indices_buffer.clear();
sparse_values_buffer.clear();
}
// test dump
auto path = dir_ + "/TestGeneral_dump";
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, streamer->dump(dumper));
ASSERT_EQ(0, streamer->close());
ASSERT_EQ(0, dumper->close());
// do searcher get vector
IndexSearcher::Pointer searcher =
IndexFactory::CreateSearcher("FlatSparseSearcher");
auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_TRUE(read_storage != nullptr);
ASSERT_TRUE(searcher != nullptr);
ASSERT_EQ(0, read_storage->open(path, false));
ASSERT_EQ(0, searcher->init(Params()));
uint32_t loop = 5;
while (loop--) {
std::cout << "loop: " << loop << std::endl;
ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
// test searcher get data
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(0, searcher->get_sparse_vector(i, &sparse_count,
&sparse_indices_buffer,
&sparse_values_buffer));
ASSERT_EQ(sparse_dim_count, sparse_count);
const uint32_t *sparse_indices_ptr =
reinterpret_cast<const uint32_t *>(sparse_indices_buffer.data());
const float *sparse_values_ptr =
reinterpret_cast<const float *>(sparse_values_buffer.data());
for (size_t j = 0; j < sparse_count; ++j) {
ASSERT_EQ(sparse_indices_ptr[j], sparse_indices_list[i][j]);
ASSERT_FLOAT_EQ(sparse_values_ptr[j], sparse_vec_list[i][j]);
// std::cout << "1: " << sparse_values_ptr[j]
// << " 2: " << sparse_vec_list[i][j] << std::endl;
}
// must clear ^_^
sparse_indices_buffer.clear();
sparse_values_buffer.clear();
}
ASSERT_EQ(searcher->unload(), 0);
}
}
TEST_F(FlatSparseSearcherTest, TestSearch) {
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("FlatSparseStreamer");
ASSERT_TRUE(streamer != nullptr);
Params params;
Params stg_params;
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch.index", true));
ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
ASSERT_EQ(0, streamer->open(storage));
size_t cnt = 5000UL;
auto ctx = streamer->create_context();
ASSERT_TRUE(!!ctx);
IndexQueryMeta qmeta(IndexMeta::DT_FP32);
NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
NumericalVector<float> sparse_velues(sparse_dim_count);
for (size_t i = 0; i < cnt; ++i) {
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = -1.0 * i - 1.0f;
}
ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), qmeta, ctx));
}
// test dump
auto path = dir_ + "/TestGeneral_dump";
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, streamer->dump(dumper));
ASSERT_EQ(0, streamer->close());
ASSERT_EQ(0, dumper->close());
// do searcher get vector
IndexSearcher::Pointer searcher =
IndexFactory::CreateSearcher("FlatSparseSearcher");
auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_TRUE(read_storage != nullptr);
ASSERT_TRUE(searcher != nullptr);
ASSERT_EQ(0, read_storage->open(path, false));
ASSERT_EQ(0, searcher->init(Params()));
ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
size_t step = 50;
for (size_t i = 0; i < cnt; i += step) {
// std::cout << "search " << i << std::endl;
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = i + 1.0f;
}
ctx->set_topk(1U);
ASSERT_EQ(0,
searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), qmeta, ctx));
auto &result1 = ctx->result();
ASSERT_EQ(1UL, result1.size());
ASSERT_EQ(0, result1[0].key());
// std::cout << result1[0].key() << " " << result1[0].score() << std::endl;
ctx->set_topk(3U);
ASSERT_EQ(0,
searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), qmeta, ctx));
auto &result2 = ctx->result();
ASSERT_EQ(3UL, result2.size());
for (size_t i = 0; i < 3UL; ++i) {
// std::cout << result2[i].key() << " " << result2[i].score() <<
// std::endl;
ASSERT_EQ(i, result2[i].key());
}
}
ctx->set_topk(100U);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = 10.1f;
}
ASSERT_EQ(0, searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), qmeta, ctx));
auto &result = ctx->result();
ASSERT_EQ(100U, result.size());
for (size_t i = 0; i < 100; ++i) {
ASSERT_EQ(i, result[i].key());
}
}
TEST_F(FlatSparseSearcherTest, TestSearchPKeys) {
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("FlatSparseStreamer");
ASSERT_TRUE(streamer != nullptr);
Params params;
Params stg_params;
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchByKeys.index", true));
ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
ASSERT_EQ(0, streamer->open(storage));
size_t cnt = 5000UL;
auto ctx = streamer->create_context();
ASSERT_TRUE(!!ctx);
IndexQueryMeta qmeta(IndexMeta::DT_FP32);
std::vector<std::vector<uint64_t>> p_keys;
p_keys.resize(1);
p_keys[0].resize(cnt);
NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
NumericalVector<float> sparse_velues(sparse_dim_count);
for (size_t i = 0; i < cnt; ++i) {
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = -1.0 * i - 1.0f;
}
ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), qmeta, ctx));
p_keys[0][i] = i;
}
// test dump
auto path = dir_ + "/TestGeneral_dump";
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, streamer->dump(dumper));
ASSERT_EQ(0, streamer->close());
ASSERT_EQ(0, dumper->close());
// do searcher get vector
IndexSearcher::Pointer searcher =
IndexFactory::CreateSearcher("FlatSparseSearcher");
auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
ASSERT_TRUE(read_storage != nullptr);
ASSERT_TRUE(searcher != nullptr);
ASSERT_EQ(0, read_storage->open(path, false));
ASSERT_EQ(0, searcher->init(Params()));
ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
size_t topk = 3;
for (size_t i = 0; i < cnt; i += 50) {
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = i + 1.0f;
}
ctx->set_topk(1U);
ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), p_keys, qmeta, ctx));
auto &result1 = ctx->result();
ASSERT_EQ(1UL, result1.size());
ASSERT_EQ(0, result1[0].key());
ctx->set_topk(topk);
ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), p_keys, qmeta, ctx));
auto &result2 = ctx->result();
ASSERT_EQ(topk, result2.size());
ASSERT_EQ(0, result2[0].key());
ASSERT_EQ(1, result2[1].key());
ASSERT_EQ(2, result2[2].key());
}
{
ctx->set_topk(100U);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = 1.0f;
}
ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), p_keys, qmeta, ctx));
auto &result = ctx->result();
ASSERT_EQ(100U, result.size());
ASSERT_EQ(0, result[0].key());
ASSERT_EQ(1, result[1].key());
ASSERT_EQ(10, result[10].key());
ASSERT_EQ(20, result[20].key());
ASSERT_EQ(30, result[30].key());
ASSERT_EQ(35, result[35].key());
ASSERT_EQ(99, result[99].key());
}
{
ctx->set_topk(100U);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = 10.0f;
}
p_keys[0] = {{cnt + 1, 10, 1, 15, cnt + 2}};
ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), p_keys, qmeta, ctx));
auto &result = ctx->result();
ASSERT_EQ(3U, result.size());
ASSERT_EQ(1, result[0].key());
ASSERT_EQ(10, result[1].key());
ASSERT_EQ(15, result[2].key());
}
{
ctx->set_topk(100U);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = 9.0f;
}
p_keys[0].clear();
for (size_t j = 0; j < cnt; j += 10) {
p_keys[0].push_back((uint64_t)j);
}
ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), p_keys, qmeta, ctx));
auto &result = ctx->result();
ASSERT_EQ(100U, result.size());
ASSERT_EQ(0, result[0].key());
ASSERT_EQ(10, result[1].key());
ASSERT_EQ(100, result[10].key());
ASSERT_EQ(200, result[20].key());
ASSERT_EQ(300, result[30].key());
ASSERT_EQ(350, result[35].key());
ASSERT_EQ(990, result[99].key());
}
}
TEST_F(FlatSparseSearcherTest, TestMultiThread) {
IndexStreamer::Pointer streamer =
IndexFactory::CreateStreamer("FlatSparseStreamer");
ASSERT_TRUE(streamer != nullptr);
Params params;
constexpr size_t static sparse_dim_count = 32;
IndexMeta meta(IndexMeta::MetaType::MT_SPARSE, IndexMeta::DataType::DT_FP32);
meta.set_metric("InnerProductSparse", 0, Params());
ASSERT_EQ(0, streamer->init(meta, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true));
ASSERT_EQ(0, streamer->open(storage));
auto addVector = [&streamer](int baseKey, size_t addCnt) {
IndexQueryMeta qmeta(IndexMeta::DT_FP32);
size_t succAdd = 0;
auto ctx = streamer->create_context();
for (size_t i = 0; i < addCnt; i++) {
NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
NumericalVector<float> sparse_velues(sparse_dim_count);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = (float)i + baseKey;
}
succAdd += !streamer->add_impl(baseKey + i, sparse_dim_count,
sparse_indices.data(),
sparse_velues.data(), qmeta, ctx);
}
streamer->flush(0UL);
return succAdd;
};
auto t2 = std::async(std::launch::async, addVector, 1000, 1000);
auto t3 = std::async(std::launch::async, addVector, 2000, 1000);
auto t1 = std::async(std::launch::async, addVector, 0, 1000);
ASSERT_EQ(1000U, t1.get());
ASSERT_EQ(1000U, t2.get());
ASSERT_EQ(1000U, t3.get());
streamer->close();
// checking data
ASSERT_EQ(0, streamer->open(storage));
auto provider = streamer->create_sparse_provider();
auto iter = provider->create_iterator();
ASSERT_TRUE(!!iter);
size_t total = 0;
uint64_t min = 1000;
uint64_t max = 0;
std::set<uint64_t> keys;
while (iter->is_valid()) {
const uint32_t sparse_count = iter->sparse_count();
ASSERT_EQ(sparse_count, sparse_dim_count);
const float *data = reinterpret_cast<const float *>(iter->sparse_data());
for (size_t j = 0; j < sparse_dim_count; ++j) {
ASSERT_EQ((float)iter->key(), data[j]);
}
total++;
min = std::min(min, iter->key());
max = std::max(max, iter->key());
keys.insert(iter->key());
iter->next();
}
ASSERT_EQ(3000, keys.size());
ASSERT_EQ(3000, total);
ASSERT_EQ(0, min);
ASSERT_EQ(2999, max);
// test dump
auto path = dir_ + "/TestGeneral_dump";
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, streamer->dump(dumper));
ASSERT_EQ(0, streamer->close());
ASSERT_EQ(0, dumper->close());
// do searcher get vector
IndexSearcher::Pointer searcher =
IndexFactory::CreateSearcher("FlatSparseSearcher");
auto read_storage = IndexFactory::CreateStorage("MMapFileReadStorage");
ASSERT_TRUE(read_storage != nullptr);
ASSERT_TRUE(searcher != nullptr);
ASSERT_EQ(0, read_storage->open(path, false));
ASSERT_EQ(0, searcher->init(Params()));
ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
// ====== multi thread search
size_t topk = 10;
size_t cnt = 3000;
auto knnSearch = [&]() {
auto linearCtx = searcher->create_context();
auto linearByPkeysCtx = searcher->create_context();
auto ctx = searcher->create_context();
IndexQueryMeta qmeta(IndexMeta::DT_FP32);
linearCtx->set_topk(topk);
linearByPkeysCtx->set_topk(topk);
ctx->set_topk(topk);
size_t totalCnts = 0;
size_t totalHits = 0;
for (size_t i = 0; i < cnt; i += 1) {
NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
NumericalVector<float> sparse_velues(sparse_dim_count);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_velues[j] = ((float)i + 1.1f);
}
ASSERT_EQ(0,
searcher->search_impl(sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), qmeta, ctx));
ASSERT_EQ(
0, searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), qmeta, linearCtx));
std::vector<std::vector<uint64_t>> p_keys = {{cnt - 1, cnt - 2, cnt - 3}};
ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
sparse_dim_count, sparse_indices.data(),
sparse_velues.data(), p_keys, qmeta, linearByPkeysCtx));
auto &r1 = ctx->result();
ASSERT_EQ(topk, r1.size());
// std::cout << "r1 top1: " << r1[0].key() << ", score: " << r1[0].score()
// << std::endl;
ASSERT_EQ(cnt - 1, r1[0].key());
auto &r2 = linearCtx->result();
ASSERT_EQ(topk, r2.size());
// std::cout << "r2 top1: " << r2[0].key() << ", score: " << r2[0].score()
// << std::endl;
ASSERT_EQ(cnt - 1, r2[0].key());
auto &r3 = linearByPkeysCtx->result();
ASSERT_EQ(std::min(topk, p_keys[0].size()), r3.size());
#if 0
printf("linear: %zd => %zd %zd %zd %zd %zd\n", i, r2[0].key,
r2[1].key, r2[2].key, r2[3].key, r2[4].key);
printf("knn: %zd => %zd %zd %zd %zd %zd\n", i, r1[0].key, r1[1].key,
r1[2].key, r1[3].key, r1[4].key);
#endif
for (size_t k = 0; k < topk; ++k) {
totalCnts++;
for (size_t j = 0; j < topk; ++j) {
if (r2[j].key() == r1[k].key()) {
totalHits++;
break;
}
}
}
}
printf("%f\n", totalHits * 1.0f / totalCnts);
ASSERT_FLOAT_EQ(1.0f, totalHits * 1.0f / totalCnts);
};
auto s1 = std::async(std::launch::async, knnSearch);
auto s2 = std::async(std::launch::async, knnSearch);
auto s3 = std::async(std::launch::async, knnSearch);
s1.wait();
s2.wait();
s3.wait();
}
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
@@ -0,0 +1,232 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string>
#include <vector>
#include <ailego/buffer/buffer_manager.h>
#include <ailego/math/norm2_matrix.h>
#include <ailego/utility/math_helper.h>
#include <ailego/utility/memory_helper.h>
#include <algorithm/flat_sparse/flat_sparse_utility.h>
#include <framework/index_framework.h>
#include <framework/index_streamer.h>
#include <gtest/gtest.h>
using namespace std;
using namespace testing;
using namespace zvec::ailego;
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
namespace zvec {
namespace core {
class FlatSparseStreamerTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
void generate_sparse_data(
size_t cnt, uint32_t sparse_dim_count,
std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
std::vector<NumericalVector<float>> &sparse_vec_list, bool norm);
static std::string dir_;
static shared_ptr<IndexMeta> index_meta_ptr_;
};
std::string FlatSparseStreamerTest::dir_("FlatSparseStreamerTest/");
shared_ptr<IndexMeta> FlatSparseStreamerTest::index_meta_ptr_;
void FlatSparseStreamerTest::generate_sparse_data(
size_t cnt, uint32_t sparse_dim_count,
std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
std::vector<NumericalVector<float>> &sparse_vec_list, bool norm) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(-1.0, 1.0);
for (size_t i = 0; i < cnt; ++i) {
// prepare sparse
NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
NumericalVector<float> sparse_vec(sparse_dim_count);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_vec[j] = dist(gen);
}
float norm;
ailego::Norm2Matrix<float, 1>::Compute(sparse_vec.data(), sparse_dim_count,
&norm);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_vec[j] = sparse_vec[j] / norm;
}
sparse_indices_list.push_back(sparse_indices);
sparse_vec_list.push_back(sparse_vec);
}
}
void FlatSparseStreamerTest::SetUp(void) {
index_meta_ptr_.reset(new (nothrow) IndexMeta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32));
index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params());
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
void FlatSparseStreamerTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
TEST_F(FlatSparseStreamerTest, TestGeneral) {
IndexStreamer::Pointer write_streamer =
IndexFactory::CreateStreamer("FlatSparseStreamer");
ASSERT_TRUE(write_streamer != nullptr);
size_t sparse_dim_count = 32;
IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32);
index_meta.set_metric("InnerProductSparse", 0, ailego::Params());
ailego::Params params;
ailego::Params stg_params;
auto write_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, write_storage->init(stg_params));
ASSERT_EQ(0, write_storage->open(dir_ + "/Test/FlatSparseSearch", true));
ASSERT_EQ(0, write_streamer->init(index_meta, params));
ASSERT_EQ(0, write_streamer->open(write_storage));
size_t cnt = 20000U;
auto ctx = write_streamer->create_context();
ASSERT_TRUE(!!ctx);
std::vector<NumericalVector<uint32_t>> sparse_indices_list;
std::vector<NumericalVector<float>> sparse_vec_list;
generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
sparse_vec_list, true);
IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32);
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(0, write_streamer->add_impl(
i, sparse_dim_count, sparse_indices_list[i].data(),
sparse_vec_list[i].data(), qmeta, ctx));
}
write_streamer->flush(0UL);
write_streamer->close();
write_streamer.reset();
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("FlatSparseStreamer");
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, read_storage);
ASSERT_EQ(0, read_storage->init(stg_params));
ASSERT_EQ(0, read_storage->open(dir_ + "/Test/FlatSparseSearch", false));
ASSERT_EQ(0, read_streamer->open(read_storage));
auto linearCtx = read_streamer->create_context();
ASSERT_TRUE(!!linearCtx);
auto knnCtx = read_streamer->create_context();
ASSERT_TRUE(!!knnCtx);
// streamer->print_debug_info();
size_t topk = 200;
linearCtx->set_topk(topk);
knnCtx->set_topk(topk);
uint64_t knnTotalTime = 0;
uint64_t linearTotalTime = 0;
int totalHits = 0;
int totalCnts = 0;
int topk1Hits = 0;
for (size_t i = 0; i < cnt; i += 100) {
const auto &sparse_indices = sparse_indices_list[i];
const auto &sparse_vec = sparse_vec_list[i];
auto t1 = ailego::Realtime::MicroSeconds();
ASSERT_EQ(
0, read_streamer->search_impl(sparse_dim_count, sparse_indices.data(),
sparse_vec.data(), qmeta, knnCtx));
auto t2 = ailego::Realtime::MicroSeconds();
ASSERT_EQ(0, read_streamer->search_bf_impl(
sparse_dim_count, sparse_indices.data(), sparse_vec.data(),
qmeta, linearCtx));
auto t3 = ailego::Realtime::MicroSeconds();
knnTotalTime += t2 - t1;
linearTotalTime += t3 - t2;
// std::cout << "i: " << i << std::endl;
auto &knnResult = knnCtx->result();
ASSERT_EQ(topk, knnResult.size());
topk1Hits += i == knnResult[0].key();
auto &linearResult = linearCtx->result();
ASSERT_EQ(topk, linearResult.size());
ASSERT_EQ(i, linearResult[0].key());
for (size_t k = 0; k < topk; ++k) {
totalCnts++;
for (size_t j = 0; j < topk; ++j) {
if (linearResult[j].key() == knnResult[k].key()) {
totalHits++;
break;
}
}
}
}
float recall = totalHits * 1.0f / totalCnts;
float topk1Recall = topk1Hits * 100.0f / cnt;
// float cost = linearTotalTime * 1.0f / knnTotalTime;
std::cout << "knnTotalTime=" << knnTotalTime
<< " linearTotalTime=" << linearTotalTime << std::endl;
#if 0
printf("knnTotalTime=%zd linearTotalTime=%zd totalHits=%d totalCnts=%d "
"R@%zd=%f R@1=%f cost=%f\n",
knnTotalTime, linearTotalTime, totalHits, totalCnts, topk, recall,
topk1Recall, cost);
#endif
EXPECT_GT(recall, 0.80f);
EXPECT_GT(topk1Recall, 0.80f);
// EXPECT_GT(cost, 2.0f);
}
} // namespace core
} // namespace zvec
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
File diff suppressed because it is too large Load Diff
+14
View File
@@ -0,0 +1,14 @@
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
foreach(CC_SRCS ${ALL_TEST_SRCS})
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
cc_gtest(
NAME ${CC_TARGET}
STRICT
LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_hnsw core_knn_flat
SRCS ${CC_SRCS}
INCS . ${CMAKE_SOURCE_DIR}/src/core ${CMAKE_SOURCE_DIR}/src/core/algorithm/hnsw
)
endforeach()
@@ -0,0 +1,543 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_builder.h"
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <future>
#include <ailego/container/vector.h>
#include <gtest/gtest.h>
#include "framework/index_framework.h"
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
using namespace std;
using namespace zvec::ailego;
namespace zvec {
namespace core {
constexpr size_t static dim = 16;
class HnswBuilderTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
static std::string _dir;
static shared_ptr<IndexMeta> _index_meta_ptr;
};
std::string HnswBuilderTest::_dir("hnswBuilderTest");
shared_ptr<IndexMeta> HnswBuilderTest::_index_meta_ptr;
void HnswBuilderTest::SetUp(void) {
_index_meta_ptr.reset(new (nothrow)
IndexMeta(IndexMeta::DataType::DT_FP32, dim));
_index_meta_ptr->set_metric("SquaredEuclidean", 0, ailego::Params());
}
void HnswBuilderTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str());
system(cmdBuf);
}
TEST_F(HnswBuilderTest, TestGeneral) {
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("HnswBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<OnePassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
ailego::Params params;
// params.set("proxima.hnsw.builder.thread_count", 1);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestGeneral";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// ASSERT_GT(stats.dumped_costtime(), 0UL);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
auto holder2 =
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder2->emplace(i, vec));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
TEST_F(HnswBuilderTest, TestMemquota) {
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("HnswBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<OnePassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
ailego::Params params;
params.set("proxima.hnsw.builder.memory_quota", 100000UL);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(IndexError_NoMemory, builder->build(holder));
}
TEST_F(HnswBuilderTest, TestIndexThreads) {
IndexBuilder::Pointer builder1 = IndexFactory::CreateBuilder("HnswBuilder");
ASSERT_NE(builder1, nullptr);
IndexBuilder::Pointer builder2 = IndexFactory::CreateBuilder("HnswBuilder");
ASSERT_NE(builder2, nullptr);
auto holder =
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
ailego::Params params;
std::srand(ailego::Realtime::MilliSeconds());
auto threads =
std::make_shared<SingleQueueIndexThreads>(std::rand() % 4, false);
ASSERT_EQ(0, builder1->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder2->init(*_index_meta_ptr, params));
auto build_index1 = [&]() {
ASSERT_EQ(0, builder1->train(threads, holder));
ASSERT_EQ(0, builder1->build(threads, holder));
};
auto build_index2 = [&]() {
ASSERT_EQ(0, builder2->train(threads, holder));
ASSERT_EQ(0, builder2->build(threads, holder));
};
auto t1 = std::async(std::launch::async, build_index1);
auto t2 = std::async(std::launch::async, build_index2);
t1.wait();
t2.wait();
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestIndexThreads";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder1->dump(dumper));
ASSERT_EQ(0, dumper->close());
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder2->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats1 = builder1->stats();
ASSERT_EQ(doc_cnt, stats1.built_count());
auto &stats2 = builder2->stats();
ASSERT_EQ(doc_cnt, stats2.built_count());
}
TEST_F(HnswBuilderTest, TestCosine) {
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("HnswBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<OnePassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, dim);
index_meta_raw.set_metric("Cosine", 0, ailego::Params());
ailego::Params converter_params;
auto converter = IndexFactory::CreateConverter("CosineFp32Converter");
converter->init(index_meta_raw, converter_params);
IndexMeta index_meta = converter->meta();
converter->transform(holder);
auto converted_holder = converter->result();
ailego::Params params;
// params.set("proxima.hnsw.builder.thread_count", 1);
ASSERT_EQ(0, builder->init(index_meta, params));
ASSERT_EQ(0, builder->train(converted_holder));
ASSERT_EQ(0, builder->build(converted_holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestCosine";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// ASSERT_GT(stats.dumped_costtime(), 0UL);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
auto holder2 =
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder2->emplace(i, vec));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
TEST_F(HnswBuilderTest, TestCosineFp16Converter) {
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("HnswBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<OnePassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, dim);
index_meta_raw.set_metric("Cosine", 0, ailego::Params());
ailego::Params converter_params;
auto converter = IndexFactory::CreateConverter("CosineFp16Converter");
converter->init(index_meta_raw, converter_params);
IndexMeta index_meta = converter->meta();
converter->transform(holder);
auto converted_holder = converter->result();
ailego::Params params;
// params.set("proxima.hnsw.builder.thread_count", 1);
ASSERT_EQ(0, builder->init(index_meta, params));
ASSERT_EQ(0, builder->train(converted_holder));
ASSERT_EQ(0, builder->build(converted_holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestCosineFp16Converter";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// ASSERT_GT(stats.dumped_costtime(), 0UL);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
auto holder2 =
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder2->emplace(i, vec));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
TEST_F(HnswBuilderTest, TestCosineInt8Converter) {
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("HnswBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<OnePassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, dim);
index_meta_raw.set_metric("Cosine", 0, ailego::Params());
ailego::Params converter_params;
auto converter = IndexFactory::CreateConverter("CosineInt8Converter");
converter->init(index_meta_raw, converter_params);
IndexMeta index_meta = converter->meta();
converter->transform(holder);
auto converted_holder = converter->result();
ailego::Params params;
// params.set("proxima.hnsw.builder.thread_count", 1);
ASSERT_EQ(0, builder->init(index_meta, params));
ASSERT_EQ(0, builder->train(converted_holder));
ASSERT_EQ(0, builder->build(converted_holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestCosineInt8Converter";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// ASSERT_GT(stats.dumped_costtime(), 0UL);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
auto holder2 =
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder2->emplace(i, vec));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
TEST_F(HnswBuilderTest, TestCosineInt4Converter) {
IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("HnswBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<OnePassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, dim);
index_meta_raw.set_metric("Cosine", 0, ailego::Params());
ailego::Params converter_params;
auto converter = IndexFactory::CreateConverter("CosineInt4Converter");
converter->init(index_meta_raw, converter_params);
IndexMeta index_meta = converter->meta();
converter->transform(holder);
auto converted_holder = converter->result();
ailego::Params params;
// params.set("proxima.hnsw.builder.thread_count", 1);
ASSERT_EQ(0, builder->init(index_meta, params));
ASSERT_EQ(0, builder->train(converted_holder));
ASSERT_EQ(0, builder->build(converted_holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestCosineInt4Converter";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// ASSERT_GT(stats.dumped_costtime(), 0UL);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
auto holder2 =
make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder2->emplace(i, vec));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
} // namespace core
} // namespace zvec
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,271 @@
#include <future>
#include <string>
#include <vector>
#include <ailego/buffer/buffer_manager.h>
#include <ailego/utility/math_helper.h>
#include <ailego/utility/memory_helper.h>
#include <algorithm/hnsw/hnsw_params.h>
#include <framework/index_framework.h>
#include <framework/index_streamer.h>
#include <gtest/gtest.h>
using namespace zvec::core;
using namespace zvec::ailego;
using namespace std;
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
constexpr size_t static dim = 16;
class HnswStreamerTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
void hybrid_scale(std::vector<float> &dense_value,
std::vector<float> &sparse_value, float alpha_scale);
static std::string dir_;
static std::shared_ptr<IndexMeta> index_meta_ptr_;
};
std::string HnswStreamerTest::dir_("streamer_test/");
std::shared_ptr<IndexMeta> HnswStreamerTest::index_meta_ptr_;
void HnswStreamerTest::SetUp(void) {
index_meta_ptr_.reset(new (std::nothrow)
IndexMeta(IndexMeta::DataType::DT_FP32, dim));
index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params());
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
void HnswStreamerTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
TEST_F(HnswStreamerTest, TestHnswSearch) {
BufferManager::Instance().init(10 * 1024 * 1024, 1);
IndexStreamer::Pointer write_streamer =
IndexFactory::CreateStreamer("HnswStreamer");
ASSERT_TRUE(write_streamer != nullptr);
Params params;
params.set(PARAM_HNSW_STREAMER_GET_VECTOR_ENABLE, true);
ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/Test/HnswSearch", true));
ASSERT_EQ(0, write_streamer->open(storage));
auto ctx = write_streamer->create_context();
ASSERT_TRUE(!!ctx);
size_t cnt = 10000UL;
IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
for (size_t i = 0; i < cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
write_streamer->add_impl(i, vec.data(), qmeta, ctx);
}
write_streamer->flush(0UL);
write_streamer->close();
write_streamer.reset();
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("HnswStreamer");
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
auto read_storage = IndexFactory::CreateStorage("BufferStorage");
ASSERT_NE(nullptr, read_storage);
ASSERT_EQ(0, read_storage->init(stg_params));
ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSearch", false));
ASSERT_EQ(0, read_streamer->open(read_storage));
size_t topk = 3;
auto provider = read_streamer->create_provider();
for (size_t i = 0; i < cnt; i += 1) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result1 = ctx->result();
ASSERT_EQ(topk, result1.size());
IndexStorage::MemoryBlock block;
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
const float *data = (float *)block.data();
for (size_t j = 0; j < dim; ++j) {
ASSERT_EQ(data[j], i);
}
ASSERT_EQ(i, result1[0].key());
for (size_t j = 0; j < dim; ++j) {
vec[j] = i + 0.1f;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result2 = ctx->result();
ASSERT_EQ(topk, result2.size());
ASSERT_EQ(i, result2[0].key());
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
}
ctx->set_topk(100U);
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = 10.1f;
}
ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
auto &result = ctx->result();
ASSERT_EQ(100U, result.size());
ASSERT_EQ(10, result[0].key());
ASSERT_EQ(11, result[1].key());
ASSERT_EQ(5, result[10].key());
ASSERT_EQ(0, result[20].key());
ASSERT_EQ(30, result[30].key());
ASSERT_EQ(35, result[35].key());
ASSERT_EQ(99, result[99].key());
ElapsedTime elapsed_time;
for (size_t i = 0; i < cnt; i += 1) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result1 = ctx->result();
ASSERT_EQ(topk, result1.size());
IndexStorage::MemoryBlock block;
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
const float *data = (float *)block.data();
for (size_t j = 0; j < dim; ++j) {
ASSERT_EQ(data[j], i);
}
ASSERT_EQ(i, result1[0].key());
for (size_t j = 0; j < dim; ++j) {
vec[j] = i + 0.1f;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result2 = ctx->result();
ASSERT_EQ(topk, result2.size());
ASSERT_EQ(i, result2[0].key());
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
}
read_streamer->close();
read_streamer.reset();
cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
}
TEST_F(HnswStreamerTest, TestHnswSearchMMap) {
IndexStreamer::Pointer write_streamer =
IndexFactory::CreateStreamer("HnswStreamer");
ASSERT_TRUE(write_streamer != nullptr);
Params params;
params.set(PARAM_HNSW_STREAMER_GET_VECTOR_ENABLE, true);
ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
auto storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage);
Params stg_params;
ASSERT_EQ(0, storage->init(stg_params));
ASSERT_EQ(0, storage->open(dir_ + "/Test/HnswSearchMMap", true));
ASSERT_EQ(0, write_streamer->open(storage));
auto ctx = write_streamer->create_context();
ASSERT_TRUE(!!ctx);
size_t cnt = 10000UL;
IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
for (size_t i = 0; i < cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
write_streamer->add_impl(i, vec.data(), qmeta, ctx);
}
write_streamer->flush(0UL);
write_streamer->close();
write_streamer.reset();
ElapsedTime elapsed_time;
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("HnswStreamer");
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, read_storage);
ASSERT_EQ(0, read_storage->init(stg_params));
ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSearchMMap", false));
ASSERT_EQ(0, read_streamer->open(read_storage));
size_t topk = 3;
auto provider = read_streamer->create_provider();
for (size_t i = 0; i < cnt; i += 1) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result1 = ctx->result();
ASSERT_EQ(topk, result1.size());
IndexStorage::MemoryBlock block;
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
const float *data = (float *)block.data();
for (size_t j = 0; j < dim; ++j) {
ASSERT_EQ(data[j], i);
}
ASSERT_EQ(i, result1[0].key());
for (size_t j = 0; j < dim; ++j) {
vec[j] = i + 0.1f;
}
ctx->set_topk(topk);
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
auto &result2 = ctx->result();
ASSERT_EQ(topk, result2.size());
ASSERT_EQ(i, result2[0].key());
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
}
ctx->set_topk(100U);
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = 10.1f;
}
ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
auto &result = ctx->result();
ASSERT_EQ(100U, result.size());
ASSERT_EQ(10, result[0].key());
ASSERT_EQ(11, result[1].key());
ASSERT_EQ(5, result[10].key());
ASSERT_EQ(0, result[20].key());
ASSERT_EQ(30, result[30].key());
ASSERT_EQ(35, result[35].key());
ASSERT_EQ(99, result[99].key());
read_streamer->close();
read_streamer.reset();
cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
}
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,14 @@
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
foreach(CC_SRCS ${ALL_TEST_SRCS})
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
cc_gtest(
NAME ${CC_TARGET}
STRICT
LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_hnsw_sparse
SRCS ${CC_SRCS}
INCS . ${CMAKE_SOURCE_DIR}/src/core ${CMAKE_SOURCE_DIR}/src/core/algorithm/hnsw_sparse
)
endforeach()
@@ -0,0 +1,470 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "hnsw_sparse_builder.h"
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <future>
#include <ailego/container/vector.h>
#include <gtest/gtest.h>
#include "framework/index_framework.h"
#include "hnsw_sparse_params.h"
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
using namespace std;
using namespace testing;
using namespace zvec::ailego;
namespace zvec {
namespace core {
class HnswSparseBuilderTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
static std::string _dir;
static shared_ptr<IndexMeta> _index_meta_ptr;
};
std::string HnswSparseBuilderTest::_dir("HnswSparseBuilderTest");
shared_ptr<IndexMeta> HnswSparseBuilderTest::_index_meta_ptr;
void HnswSparseBuilderTest::SetUp(void) {
_index_meta_ptr.reset(new (nothrow) IndexMeta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32));
_index_meta_ptr->set_metric("InnerProductSparse", 0, ailego::Params());
}
void HnswSparseBuilderTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str());
system(cmdBuf);
}
TEST_F(HnswSparseBuilderTest, TestGeneral) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswSparseBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<OnePassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
uint32_t sparse_count = 4;
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder->emplace(i, vec));
}
ailego::Params params;
params.set(PARAM_HNSW_SPARSE_BUILDER_THREAD_COUNT, 1);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(0, builder->build(holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestGeneral";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// ASSERT_GT(stats.dumped_costtime(), 0UL);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
auto holder2 =
make_shared<MultiPassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder2->emplace(i, vec));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
TEST_F(HnswSparseBuilderTest, TestMemquota) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswSparseBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<OnePassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
size_t doc_cnt = 1000UL;
uint32_t sparse_count = 32;
for (size_t i = 0; i < doc_cnt; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder->emplace(i, vec));
}
ailego::Params params;
params.set("proxima.hnsw.sparse_builder.memory_quota", 100000UL);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder));
ASSERT_EQ(IndexError_NoMemory, builder->build(holder));
}
TEST_F(HnswSparseBuilderTest, TestIndexThreads) {
IndexBuilder::Pointer builder1 =
IndexFactory::CreateBuilder("HnswSparseBuilder");
ASSERT_NE(builder1, nullptr);
IndexBuilder::Pointer builder2 =
IndexFactory::CreateBuilder("HnswSparseBuilder");
ASSERT_NE(builder2, nullptr);
auto holder =
make_shared<MultiPassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
size_t doc_cnt = 1000UL;
uint32_t sparse_count = 32;
for (size_t i = 0; i < doc_cnt; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder->emplace(i, vec));
}
ailego::Params params;
std::srand(ailego::Realtime::MilliSeconds());
auto threads =
std::make_shared<SingleQueueIndexThreads>(std::rand() % 4, false);
ASSERT_EQ(0, builder1->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder2->init(*_index_meta_ptr, params));
auto build_index1 = [&]() {
ASSERT_EQ(0, builder1->train(threads, holder));
ASSERT_EQ(0, builder1->build(threads, holder));
};
auto build_index2 = [&]() {
ASSERT_EQ(0, builder2->train(threads, holder));
ASSERT_EQ(0, builder2->build(threads, holder));
};
auto t1 = std::async(std::launch::async, build_index1);
auto t2 = std::async(std::launch::async, build_index2);
t1.wait();
t2.wait();
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestIndexThreads";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder1->dump(dumper));
ASSERT_EQ(0, dumper->close());
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder2->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats1 = builder1->stats();
ASSERT_EQ(doc_cnt, stats1.built_count());
auto &stats2 = builder2->stats();
ASSERT_EQ(doc_cnt, stats2.built_count());
}
TEST_F(HnswSparseBuilderTest, TestHalfFloatConverter) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswSparseBuilder");
ASSERT_NE(builder, nullptr);
auto holder =
make_shared<OnePassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
uint32_t sparse_count = 4;
size_t doc_cnt = 1000UL;
for (size_t i = 0; i < doc_cnt; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder->emplace(i, vec));
}
ailego::Params converter_params;
auto converter = IndexFactory::CreateConverter("HalfFloatSparseConverter");
converter->init(*_index_meta_ptr, converter_params);
IndexMeta index_meta = converter->meta();
converter->transform(holder);
auto converted_holder = converter->sparse_result();
ailego::Params params;
params.set(PARAM_HNSW_SPARSE_BUILDER_THREAD_COUNT, 1);
ASSERT_EQ(0, builder->init(index_meta, converter_params));
ASSERT_EQ(0, builder->train(converted_holder));
ASSERT_EQ(0, builder->build(converted_holder));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestHalFloatConverter";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// ASSERT_GT(stats.dumped_costtime(), 0UL);
// cleanup and rebuild
ASSERT_EQ(0, builder->cleanup());
auto holder2 =
make_shared<MultiPassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
size_t doc_cnt2 = 2000UL;
for (size_t i = 0; i < doc_cnt2; i++) {
SparseVector<float> vec;
NumericalVector<uint32_t> sparse_indices(sparse_count);
NumericalVector<float> sparse_values(sparse_count);
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices[j] = 20 * j;
sparse_values[j] = i;
}
vec.add_sparses(sparse_indices, sparse_values);
ASSERT_TRUE(holder2->emplace(i, vec));
}
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->train(holder2));
ASSERT_EQ(0, builder->build(holder2));
auto dumper2 = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper2, nullptr);
ASSERT_EQ(0, dumper2->create(path));
ASSERT_EQ(0, builder->dump(dumper2));
ASSERT_EQ(0, dumper2->close());
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt2, stats.built_count());
ASSERT_EQ(doc_cnt2, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
}
TEST_F(HnswSparseBuilderTest, TestIndptr) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswSparseBuilder");
ASSERT_NE(builder, nullptr);
uint32_t sparse_count = 4;
size_t doc_cnt = 1000UL;
std::vector<uint64_t> keys;
keys.reserve(doc_cnt);
std::vector<uint64_t> sparse_indptr;
sparse_indptr.reserve(doc_cnt + 1);
std::vector<uint32_t> sparse_indices;
sparse_indices.reserve(doc_cnt * sparse_count);
std::vector<float> sparse_values;
sparse_values.reserve(doc_cnt * sparse_count);
size_t sparse_count_total = 0;
sparse_indptr.push_back(0);
for (size_t i = 0; i < doc_cnt; i++) {
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices.push_back(20 * j);
sparse_values.push_back(i);
}
keys.push_back(i);
sparse_count_total += sparse_count;
sparse_indptr.push_back(sparse_count_total);
}
ailego::Params params;
params.set(PARAM_HNSW_SPARSE_BUILDER_THREAD_COUNT, 1);
ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
ASSERT_EQ(0, builder->build(doc_cnt, keys.data(), sparse_indptr.data(),
sparse_indices.data(), sparse_values.data()));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestIndptr";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// ASSERT_GT(stats.dumped_costtime(), 0UL);
}
TEST_F(HnswSparseBuilderTest, TestIndptrFp16) {
IndexBuilder::Pointer builder =
IndexFactory::CreateBuilder("HnswSparseBuilder");
ASSERT_NE(builder, nullptr);
uint32_t sparse_count = 4;
size_t doc_cnt = 1000UL;
std::vector<uint64_t> keys;
keys.reserve(doc_cnt);
std::vector<uint64_t> sparse_indptr;
sparse_indptr.reserve(doc_cnt + 1);
std::vector<uint32_t> sparse_indices;
sparse_indices.reserve(doc_cnt * sparse_count);
std::vector<float> sparse_values;
sparse_values.reserve(doc_cnt * sparse_count);
size_t sparse_count_total = 0;
sparse_indptr.push_back(0);
for (size_t i = 0; i < doc_cnt; i++) {
for (size_t j = 0; j < sparse_count; ++j) {
sparse_indices.push_back(20 * j);
sparse_values.push_back(i);
}
keys.push_back(i);
sparse_count_total += sparse_count;
sparse_indptr.push_back(sparse_count_total);
}
IndexMeta meta(IndexMeta::MetaType::MT_SPARSE, IndexMeta::DataType::DT_FP16);
ailego::Params params;
params.set(PARAM_HNSW_SPARSE_BUILDER_THREAD_COUNT, 1);
ASSERT_EQ(0, builder->init(meta, params));
IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32);
ASSERT_EQ(0, builder->build(qmeta, doc_cnt, keys.data(), sparse_indptr.data(),
sparse_indices.data(), sparse_values.data()));
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
string path = _dir + "/TestIndptrFp16";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats = builder->stats();
ASSERT_EQ(0UL, stats.trained_count());
ASSERT_EQ(doc_cnt, stats.built_count());
ASSERT_EQ(doc_cnt, stats.dumped_count());
ASSERT_EQ(0UL, stats.discarded_count());
ASSERT_EQ(0UL, stats.trained_costtime());
ASSERT_GT(stats.built_costtime(), 0UL);
// ASSERT_GT(stats.dumped_costtime(), 0UL);
}
} // namespace core
} // namespace zvec
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,361 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <future>
#include <iostream>
#include <memory>
#include <ailego/container/vector.h>
#include <ailego/math/norm_matrix.h>
#include <gtest/gtest.h>
#include "hnsw_sparse_streamer.h"
using namespace std;
using namespace testing;
using namespace zvec::ailego;
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#endif
namespace zvec {
namespace core {
class HnswSparseStreamerTest : public testing::Test {
protected:
void SetUp(void);
void TearDown(void);
void generate_sparse_data(
size_t cnt, uint32_t sparse_dim_count,
std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
std::vector<NumericalVector<float>> &sparse_vec_list, bool norm);
static std::string dir_;
static shared_ptr<IndexMeta> index_meta_ptr_;
};
std::string HnswSparseStreamerTest::dir_("HnswSparseStreamerTest/");
shared_ptr<IndexMeta> HnswSparseStreamerTest::index_meta_ptr_;
void HnswSparseStreamerTest::generate_sparse_data(
size_t cnt, uint32_t sparse_dim_count,
std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
std::vector<NumericalVector<float>> &sparse_vec_list, bool norm) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<float> dist(-1.0, 1.0);
for (size_t i = 0; i < cnt; ++i) {
// prepare sparse
NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
NumericalVector<float> sparse_vec(sparse_dim_count);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_indices[j] = j * 20;
sparse_vec[j] = dist(gen);
}
float norm;
ailego::Norm2Matrix<float, 1>::Compute(sparse_vec.data(), sparse_dim_count,
&norm);
for (size_t j = 0; j < sparse_dim_count; ++j) {
sparse_vec[j] = sparse_vec[j] / norm;
}
sparse_indices_list.push_back(sparse_indices);
sparse_vec_list.push_back(sparse_vec);
}
}
void HnswSparseStreamerTest::SetUp(void) {
index_meta_ptr_.reset(new (nothrow) IndexMeta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32));
index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params());
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
void HnswSparseStreamerTest::TearDown(void) {
char cmdBuf[100];
snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str());
system(cmdBuf);
}
TEST_F(HnswSparseStreamerTest, TestGeneral) {
BufferManager::Instance().init(10 * 1024 * 1024, 1);
IndexStreamer::Pointer write_streamer =
IndexFactory::CreateStreamer("HnswSparseStreamer");
ASSERT_TRUE(write_streamer != nullptr);
size_t sparse_dim_count = 32;
IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32);
index_meta.set_metric("InnerProductSparse", 0, ailego::Params());
ailego::Params params;
params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 20);
params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 16);
params.set(PARAM_HNSW_SPARSE_STREAMER_EFCONSTRUCTION, 10);
params.set(PARAM_HNSW_SPARSE_STREAMER_EF, 5);
params.set(PARAM_HNSW_SPARSE_STREAMER_BRUTE_FORCE_THRESHOLD, 1000U);
ailego::Params stg_params;
auto write_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, write_storage->init(stg_params));
ASSERT_EQ(0, write_storage->open(dir_ + "/Test/HnswSparseSearch", true));
ASSERT_EQ(0, write_streamer->init(index_meta, params));
ASSERT_EQ(0, write_streamer->open(write_storage));
size_t cnt = 20000U;
auto ctx = write_streamer->create_context();
ASSERT_TRUE(!!ctx);
std::vector<NumericalVector<uint32_t>> sparse_indices_list;
std::vector<NumericalVector<float>> sparse_vec_list;
generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
sparse_vec_list, true);
IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32);
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(0, write_streamer->add_impl(
i, sparse_dim_count, sparse_indices_list[i].data(),
sparse_vec_list[i].data(), qmeta, ctx));
}
write_streamer->flush(0UL);
write_streamer->close();
write_streamer.reset();
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("HnswSparseStreamer");
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
auto read_storage = IndexFactory::CreateStorage("BufferStorage");
ASSERT_NE(nullptr, read_storage);
ASSERT_EQ(0, read_storage->init(stg_params));
ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSparseSearch", false));
ASSERT_EQ(0, read_streamer->open(read_storage));
auto linearCtx = read_streamer->create_context();
ASSERT_TRUE(!!linearCtx);
auto knnCtx = read_streamer->create_context();
ASSERT_TRUE(!!knnCtx);
// streamer->print_debug_info();
size_t topk = 200;
linearCtx->set_topk(topk);
knnCtx->set_topk(topk);
uint64_t knnTotalTime = 0;
uint64_t linearTotalTime = 0;
int totalHits = 0;
int totalCnts = 0;
int topk1Hits = 0;
for (size_t i = 0; i < cnt; i += 100) {
const auto &sparse_indices = sparse_indices_list[i];
const auto &sparse_vec = sparse_vec_list[i];
auto t1 = ailego::Realtime::MicroSeconds();
ASSERT_EQ(
0, read_streamer->search_impl(sparse_dim_count, sparse_indices.data(),
sparse_vec.data(), qmeta, knnCtx));
auto t2 = ailego::Realtime::MicroSeconds();
ASSERT_EQ(0, read_streamer->search_bf_impl(
sparse_dim_count, sparse_indices.data(), sparse_vec.data(),
qmeta, linearCtx));
auto t3 = ailego::Realtime::MicroSeconds();
knnTotalTime += t2 - t1;
linearTotalTime += t3 - t2;
// std::cout << "i: " << i << std::endl;
auto &knnResult = knnCtx->result();
ASSERT_EQ(topk, knnResult.size());
topk1Hits += i == knnResult[0].key();
auto &linearResult = linearCtx->result();
ASSERT_EQ(topk, linearResult.size());
ASSERT_EQ(i, linearResult[0].key());
for (size_t k = 0; k < topk; ++k) {
totalCnts++;
for (size_t j = 0; j < topk; ++j) {
if (linearResult[j].key() == knnResult[k].key()) {
totalHits++;
break;
}
}
}
}
float recall = totalHits * 1.0f / totalCnts;
float topk1Recall = topk1Hits * 100.0f / cnt;
float cost = linearTotalTime * 1.0f / knnTotalTime;
#if 0
printf("knnTotalTime=%zd linearTotalTime=%zd totalHits=%d totalCnts=%d "
"R@%zd=%f R@1=%f cost=%f\n",
knnTotalTime, linearTotalTime, totalHits, totalCnts, topk, recall,
topk1Recall, cost);
#endif
EXPECT_GT(recall, 0.80f);
EXPECT_GT(topk1Recall, 0.80f);
EXPECT_GT(cost, 2.0f);
}
TEST_F(HnswSparseStreamerTest, TestHnswSearchMMap) {
IndexStreamer::Pointer write_streamer =
IndexFactory::CreateStreamer("HnswSparseStreamer");
ASSERT_TRUE(write_streamer != nullptr);
size_t sparse_dim_count = 32;
IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32);
index_meta.set_metric("InnerProductSparse", 0, ailego::Params());
ailego::Params params;
params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 20);
params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 16);
params.set(PARAM_HNSW_SPARSE_STREAMER_EFCONSTRUCTION, 10);
params.set(PARAM_HNSW_SPARSE_STREAMER_EF, 5);
params.set(PARAM_HNSW_SPARSE_STREAMER_BRUTE_FORCE_THRESHOLD, 1000U);
ailego::Params stg_params;
auto write_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, write_storage->init(stg_params));
ASSERT_EQ(0, write_storage->open(dir_ + "/Test/HnswSparseSearch", true));
ASSERT_EQ(0, write_streamer->init(index_meta, params));
ASSERT_EQ(0, write_streamer->open(write_storage));
size_t cnt = 20000U;
auto ctx = write_streamer->create_context();
ASSERT_TRUE(!!ctx);
std::vector<NumericalVector<uint32_t>> sparse_indices_list;
std::vector<NumericalVector<float>> sparse_vec_list;
generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
sparse_vec_list, true);
IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE,
IndexMeta::DataType::DT_FP32);
for (size_t i = 0; i < cnt; i++) {
ASSERT_EQ(0, write_streamer->add_impl(
i, sparse_dim_count, sparse_indices_list[i].data(),
sparse_vec_list[i].data(), qmeta, ctx));
}
write_streamer->flush(0UL);
write_streamer->close();
write_streamer.reset();
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("HnswSparseStreamer");
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, read_storage);
ASSERT_EQ(0, read_storage->init(stg_params));
ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSparseSearch", false));
ASSERT_EQ(0, read_streamer->open(read_storage));
auto linearCtx = read_streamer->create_context();
ASSERT_TRUE(!!linearCtx);
auto knnCtx = read_streamer->create_context();
ASSERT_TRUE(!!knnCtx);
// streamer->print_debug_info();
size_t topk = 200;
linearCtx->set_topk(topk);
knnCtx->set_topk(topk);
uint64_t knnTotalTime = 0;
uint64_t linearTotalTime = 0;
int totalHits = 0;
int totalCnts = 0;
int topk1Hits = 0;
for (size_t i = 0; i < cnt; i += 100) {
const auto &sparse_indices = sparse_indices_list[i];
const auto &sparse_vec = sparse_vec_list[i];
auto t1 = ailego::Realtime::MicroSeconds();
ASSERT_EQ(
0, read_streamer->search_impl(sparse_dim_count, sparse_indices.data(),
sparse_vec.data(), qmeta, knnCtx));
auto t2 = ailego::Realtime::MicroSeconds();
ASSERT_EQ(0, read_streamer->search_bf_impl(
sparse_dim_count, sparse_indices.data(), sparse_vec.data(),
qmeta, linearCtx));
auto t3 = ailego::Realtime::MicroSeconds();
knnTotalTime += t2 - t1;
linearTotalTime += t3 - t2;
// std::cout << "i: " << i << std::endl;
auto &knnResult = knnCtx->result();
ASSERT_EQ(topk, knnResult.size());
topk1Hits += i == knnResult[0].key();
auto &linearResult = linearCtx->result();
ASSERT_EQ(topk, linearResult.size());
ASSERT_EQ(i, linearResult[0].key());
for (size_t k = 0; k < topk; ++k) {
totalCnts++;
for (size_t j = 0; j < topk; ++j) {
if (linearResult[j].key() == knnResult[k].key()) {
totalHits++;
break;
}
}
}
}
float recall = totalHits * 1.0f / totalCnts;
float topk1Recall = topk1Hits * 100.0f / cnt;
float cost = linearTotalTime * 1.0f / knnTotalTime;
#if 0
printf("knnTotalTime=%zd linearTotalTime=%zd totalHits=%d totalCnts=%d "
"R@%zd=%f R@1=%f cost=%f\n",
knnTotalTime, linearTotalTime, totalHits, totalCnts, topk, recall,
topk1Recall, cost);
#endif
EXPECT_GT(recall, 0.80f);
EXPECT_GT(topk1Recall, 0.80f);
EXPECT_GT(cost, 2.0f);
}
} // namespace core
} // namespace zvec
#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
File diff suppressed because it is too large Load Diff
+14
View File
@@ -0,0 +1,14 @@
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
foreach(CC_SRCS ${ALL_TEST_SRCS})
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
cc_gtest(
NAME ${CC_TARGET}
STRICT
LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_cluster core_knn_flat core_knn_ivf
SRCS ${CC_SRCS}
INCS . ${CMAKE_SOURCE_DIR}/src/core ${CMAKE_SOURCE_DIR}/src/core/algorithm/ivf
)
endforeach()
@@ -0,0 +1,464 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ivf_builder.h"
#include <future>
#include <iostream>
#include <vector>
#include <ailego/container/vector.h>
#include <gtest/gtest.h>
using namespace zvec::core;
using namespace zvec::ailego;
using namespace std;
class IVFBuilderTest : public testing::Test {
protected:
void SetUp();
void TearDown();
void prepare_index_holder(uint32_t base_key, uint32_t num);
IndexMeta index_meta_;
Params params_;
uint32_t dimension_;
IndexHolder::Pointer holder_;
IndexThreads::Pointer threads_{};
};
void IVFBuilderTest::SetUp() {
dimension_ = 8U;
index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
index_meta_.set_metric("SquaredEuclidean", 0, Params());
params_.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "8");
params_.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster");
std::mt19937 gen((std::random_device())());
bool v = std::uniform_int_distribution<size_t>(0, 1)(gen);
if (v) {
threads_ = std::make_shared<SingleQueueIndexThreads>();
}
}
void IVFBuilderTest::TearDown() {}
void IVFBuilderTest::prepare_index_holder(uint32_t base_key, uint32_t num) {
MultiPassIndexHolder<IndexMeta::DataType::DT_FP32> *holder =
new MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>(dimension_);
uint32_t key = base_key;
for (size_t i = 0; i < num; ++i) {
NumericalVector<float> vec(dimension_);
for (size_t j = 0; j < dimension_; ++j) {
vec[j] = 1.0f * i;
}
holder->emplace(key + i, vec);
}
holder_.reset(holder);
}
TEST_F(IVFBuilderTest, TestInitSuccess) {
IVFBuilder builder;
int ret = builder.init(index_meta_, params_);
EXPECT_EQ(0, ret);
}
TEST_F(IVFBuilderTest, TestInitFailedWithInvalidMetric) {
IVFBuilder builder;
index_meta_.set_metric("invalid", 0, Params());
int ret = builder.init(index_meta_, params_);
EXPECT_EQ(IndexError_NoExist, ret);
}
TEST_F(IVFBuilderTest, TestInitFailedWithInvalidCentroidsNum) {
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "2");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
ret = builder.train(threads_, holder_);
EXPECT_EQ(IndexError_InvalidArgument, ret);
}
TEST_F(IVFBuilderTest, TestTrainWithHolder1Level) {
IVFBuilder builder;
int ret = builder.init(index_meta_, params_);
EXPECT_EQ(0, ret);
prepare_index_holder(0, 1000);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
auto centroid_index = builder.centroid_index();
EXPECT_GT(centroid_index->centroids_count(), 0u);
}
TEST_F(IVFBuilderTest, TestTrainWithHolder2Level) {
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
prepare_index_holder(0, 1000);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
auto centroid_index = builder.centroid_index();
EXPECT_EQ(centroid_index->centroids_count(), 8);
}
TEST_F(IVFBuilderTest, TestTrainWithTrainer2Level) {
IndexTrainer::Pointer trainer =
IndexFactory::CreateTrainer("StratifiedClusterTrainer");
ASSERT_TRUE(!!trainer);
prepare_index_holder(0, 1000);
Params params;
params.set("proxima.stratified.trainer.cluster_count", "4*2");
ASSERT_EQ(0, trainer->init(index_meta_, params));
ASSERT_EQ(0, trainer->train(threads_, holder_));
IVFBuilder builder;
int ret = builder.init(index_meta_, params_);
EXPECT_EQ(0, ret);
ret = builder.train(trainer);
EXPECT_EQ(0, ret);
auto centroid_index = builder.centroid_index();
EXPECT_EQ(centroid_index->centroids_count(), 8);
}
TEST_F(IVFBuilderTest, TestTrainWithTrainer1Level) {
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster");
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
IndexTrainer::Pointer trainer =
IndexFactory::CreateTrainer("StratifiedClusterTrainer");
ASSERT_TRUE(!!trainer);
prepare_index_holder(0, 1000);
Params params1;
params1.set("proxima.stratified.trainer.cluster_count", "4");
ASSERT_EQ(0, trainer->init(index_meta_, params1));
ASSERT_EQ(0, trainer->train(threads_, holder_));
ret = builder.train(trainer);
EXPECT_EQ(0, ret);
auto centroid_index = builder.centroid_index();
EXPECT_EQ(centroid_index->centroids_count(), 4);
}
TEST_F(IVFBuilderTest, TestBuildWith2Level) {
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
prepare_index_holder(0, 1000);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(0, ret);
EXPECT_EQ((size_t)1000, builder.stats().built_count());
}
TEST_F(IVFBuilderTest, TestBuildWith1Level) {
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster");
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
prepare_index_holder(0, 1000);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(0, ret);
EXPECT_EQ((size_t)1000, builder.stats().built_count());
}
TEST_F(IVFBuilderTest, TestDump) {
IVFBuilder builder;
int ret = builder.init(index_meta_, params_);
EXPECT_EQ(0, ret);
prepare_index_holder(0, 1000);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(0, ret);
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
ret = dumper->create("path");
EXPECT_EQ(0, ret);
ret = builder.dump(dumper);
EXPECT_EQ((size_t)1000, builder.stats().built_count());
EXPECT_EQ((size_t)1000, builder.stats().dumped_count());
EXPECT_EQ((size_t)0, builder.stats().discarded_count());
}
#if 0
TEST_F(IVFBuilderTest, TestBuildWithNoEnoughMemory)
{
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
dimension_ = 256;
index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
prepare_index_holder(0, 1000);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(IndexError_IndexFull, ret);
}
#endif
TEST_F(IVFBuilderTest, TestBuildWithEnoughMemory) {
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
dimension_ = 256;
index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
prepare_index_holder(0, 1000);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(0, ret);
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
ret = dumper->create("path");
EXPECT_EQ(0, ret);
ret = builder.dump(dumper);
EXPECT_EQ((size_t)1000, builder.stats().built_count());
EXPECT_EQ((size_t)1000, builder.stats().dumped_count());
EXPECT_EQ((size_t)0, builder.stats().discarded_count());
}
#if 0
TEST_F(IVFBuilderTest, TestBuildWithRowMajorAndNoEnoughMemory)
{
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
dimension_ = 256;
index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
index_meta_.set_major_order(IndexMeta::MajorOrder::MO_ROW);
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
prepare_index_holder(0, 1000);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(IndexError_IndexFull, ret);
}
#endif
TEST_F(IVFBuilderTest, TestBuildWithRowMajorAndMemory) {
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
dimension_ = 256;
index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
index_meta_.set_major_order(IndexMeta::MajorOrder::MO_ROW);
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
prepare_index_holder(0, 1000);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(0, ret);
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
ret = dumper->create("path");
EXPECT_EQ(0, ret);
ret = builder.dump(dumper);
EXPECT_EQ((size_t)1000, builder.stats().built_count());
EXPECT_EQ((size_t)1000, builder.stats().dumped_count());
EXPECT_EQ((size_t)0, builder.stats().discarded_count());
}
TEST_F(IVFBuilderTest, TestBuildWithEmptyCentroid) {
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "2*2");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
dimension_ = 256;
index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
index_meta_.set_major_order(IndexMeta::MajorOrder::MO_ROW);
int ret = builder.init(index_meta_, params);
EXPECT_EQ(0, ret);
size_t doc_cnt = 10;
MultiPassIndexHolder<IndexMeta::DataType::DT_FP32> *holder =
new MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>(dimension_);
for (size_t i = 0; i < doc_cnt; ++i) {
NumericalVector<float> vec(dimension_);
for (size_t j = 0; j < dimension_; ++j) {
vec[j] = 1.0f;
}
holder->emplace(i, vec);
}
holder_.reset(holder);
ret = builder.train(threads_, holder_);
EXPECT_EQ(0, ret);
ret = builder.build(threads_, holder_);
EXPECT_EQ(0, ret);
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
ret = dumper->create("path");
EXPECT_EQ(0, ret);
ret = builder.dump(dumper);
EXPECT_EQ((size_t)10, builder.stats().built_count());
EXPECT_EQ((size_t)10, builder.stats().dumped_count());
EXPECT_EQ((size_t)0, builder.stats().discarded_count());
}
TEST_F(IVFBuilderTest, TestTrainClusterParams) {
IVFBuilder builder;
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "2*2");
params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster");
prepare_index_holder(0, 1000);
EXPECT_EQ(0, builder.init(index_meta_, params));
EXPECT_EQ(0, builder.train(threads_, holder_));
EXPECT_EQ(0, builder.build(threads_, holder_));
IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
EXPECT_EQ(0, dumper->create("test.index"));
EXPECT_EQ(0, builder.dump(dumper));
}
TEST_F(IVFBuilderTest, TestIndexThreads) {
IndexBuilder::Pointer builder1 = IndexFactory::CreateBuilder("IVFBuilder");
ASSERT_NE(builder1, nullptr);
IndexBuilder::Pointer builder2 = IndexFactory::CreateBuilder("IVFBuilder");
ASSERT_NE(builder2, nullptr);
size_t dim = 128UL;
IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
std::srand(Realtime::MilliSeconds());
auto holder =
std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
size_t doc_cnt = 1000;
for (size_t i = 0; i < doc_cnt; i++) {
NumericalVector<float> vec(dim);
for (size_t j = 0; j < dim; ++j) {
vec[j] = i;
}
ASSERT_TRUE(holder->emplace(i, vec));
}
Params params;
params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "2*2");
ASSERT_EQ(0, builder1->init(meta, params));
ASSERT_EQ(0, builder2->init(meta, params));
auto threads =
std::make_shared<SingleQueueIndexThreads>(std::rand() % 4, false);
auto build_index1 = [&]() {
ASSERT_EQ(0, builder1->train(threads, holder));
ASSERT_EQ(0, builder1->build(threads, holder));
};
auto build_index2 = [&]() {
ASSERT_EQ(0, builder2->train(threads, holder));
ASSERT_EQ(0, builder2->build(threads, holder));
};
auto t1 = std::async(std::launch::async, build_index1);
auto t2 = std::async(std::launch::async, build_index2);
t1.wait();
t2.wait();
auto dumper = IndexFactory::CreateDumper("FileDumper");
ASSERT_NE(dumper, nullptr);
std::string path = "./hc_index";
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder1->dump(dumper));
ASSERT_EQ(0, dumper->close());
ASSERT_EQ(0, dumper->create(path));
ASSERT_EQ(0, builder2->dump(dumper));
ASSERT_EQ(0, dumper->close());
auto &stats1 = builder1->stats();
ASSERT_EQ(doc_cnt, stats1.built_count());
auto &stats2 = builder2->stats();
ASSERT_EQ(doc_cnt, stats2.built_count());
}
File diff suppressed because it is too large Load Diff