feat: buffer storage write (#414)
This commit is contained in:
@@ -168,6 +168,251 @@ TEST_F(FlatStreamerTest, TestLinearSearch) {
|
||||
read_streamer.reset();
|
||||
}
|
||||
|
||||
TEST_F(FlatStreamerTest, TestLinearSearchBuffer) {
|
||||
MemoryLimitPool::get_instance().init(2 * 1024UL * 1024UL * 1024UL);
|
||||
IndexStreamer::Pointer write_streamer =
|
||||
IndexFactory::CreateStreamer("FlatStreamer");
|
||||
ASSERT_TRUE(write_streamer != nullptr);
|
||||
|
||||
Params params;
|
||||
ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
|
||||
auto storage = IndexFactory::CreateStorage("BufferStorage");
|
||||
ASSERT_NE(nullptr, storage);
|
||||
Params stg_params;
|
||||
ASSERT_EQ(0, storage->init(stg_params));
|
||||
ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearchBuffer", true));
|
||||
ASSERT_EQ(0, write_streamer->open(storage));
|
||||
|
||||
auto ctx = write_streamer->create_context();
|
||||
ASSERT_TRUE(!!ctx);
|
||||
|
||||
size_t cnt = 10000UL;
|
||||
IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
|
||||
for (size_t i = 0; i < cnt; i++) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
write_streamer->add_impl(i, vec.data(), qmeta, ctx);
|
||||
}
|
||||
write_streamer->flush(0UL);
|
||||
write_streamer->close();
|
||||
write_streamer.reset();
|
||||
storage->close();
|
||||
|
||||
IndexStreamer::Pointer read_streamer =
|
||||
IndexFactory::CreateStreamer("FlatStreamer");
|
||||
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
|
||||
auto read_storage = IndexFactory::CreateStorage("BufferStorage");
|
||||
ASSERT_NE(nullptr, read_storage);
|
||||
ASSERT_EQ(0, read_storage->init(stg_params));
|
||||
ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearchBuffer", false));
|
||||
ASSERT_EQ(0, read_streamer->open(read_storage));
|
||||
size_t topk = 3;
|
||||
auto provider = read_streamer->create_provider();
|
||||
for (size_t i = 0; i < cnt; i += 1) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ctx->set_topk(topk);
|
||||
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
|
||||
auto &result1 = ctx->result();
|
||||
ASSERT_EQ(topk, result1.size());
|
||||
IndexStorage::MemoryBlock block;
|
||||
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
|
||||
const float *data = (float *)block.data();
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
ASSERT_FLOAT_EQ(data[j], i);
|
||||
}
|
||||
ASSERT_EQ(i, result1[0].key());
|
||||
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i + 0.1f;
|
||||
}
|
||||
ctx->set_topk(topk);
|
||||
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
|
||||
auto &result2 = ctx->result();
|
||||
ASSERT_EQ(topk, result2.size());
|
||||
ASSERT_EQ(i, result2[0].key());
|
||||
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
|
||||
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
|
||||
}
|
||||
|
||||
ctx->set_topk(100U);
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = 10.1f;
|
||||
}
|
||||
ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
|
||||
auto &result = ctx->result();
|
||||
ASSERT_EQ(100U, result.size());
|
||||
ASSERT_EQ(10, result[0].key());
|
||||
ASSERT_EQ(11, result[1].key());
|
||||
ASSERT_EQ(5, result[10].key());
|
||||
ASSERT_EQ(0, result[20].key());
|
||||
ASSERT_EQ(30, result[30].key());
|
||||
ASSERT_EQ(35, result[35].key());
|
||||
ASSERT_EQ(99, result[99].key());
|
||||
|
||||
ElapsedTime elapsed_time;
|
||||
for (size_t i = 0; i < cnt; i += 1) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ctx->set_topk(topk);
|
||||
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
|
||||
auto &result1 = ctx->result();
|
||||
ASSERT_EQ(topk, result1.size());
|
||||
IndexStorage::MemoryBlock block;
|
||||
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
|
||||
const float *data = (float *)block.data();
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
ASSERT_FLOAT_EQ(data[j], i);
|
||||
}
|
||||
ASSERT_EQ(i, result1[0].key());
|
||||
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i + 0.1f;
|
||||
}
|
||||
ctx->set_topk(topk);
|
||||
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
|
||||
auto &result2 = ctx->result();
|
||||
ASSERT_EQ(topk, result2.size());
|
||||
ASSERT_EQ(i, result2[0].key());
|
||||
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
|
||||
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
|
||||
}
|
||||
cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
|
||||
|
||||
read_streamer->close();
|
||||
read_streamer.reset();
|
||||
}
|
||||
|
||||
TEST_F(FlatStreamerTest, TestLinearSearchBufferMMap) {
|
||||
MemoryLimitPool::get_instance().init(2 * 1024UL * 1024UL * 1024UL);
|
||||
IndexStreamer::Pointer write_streamer =
|
||||
IndexFactory::CreateStreamer("FlatStreamer");
|
||||
ASSERT_TRUE(write_streamer != nullptr);
|
||||
|
||||
Params params;
|
||||
ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
|
||||
auto storage = IndexFactory::CreateStorage("BufferStorage");
|
||||
ASSERT_NE(nullptr, storage);
|
||||
Params stg_params;
|
||||
ASSERT_EQ(0, storage->init(stg_params));
|
||||
ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearchBuffer", true));
|
||||
ASSERT_EQ(0, write_streamer->open(storage));
|
||||
|
||||
auto ctx = write_streamer->create_context();
|
||||
ASSERT_TRUE(!!ctx);
|
||||
|
||||
size_t cnt = 10000UL;
|
||||
IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
|
||||
for (size_t i = 0; i < cnt; i++) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
write_streamer->add_impl(i, vec.data(), qmeta, ctx);
|
||||
}
|
||||
write_streamer->flush(0UL);
|
||||
write_streamer->close();
|
||||
write_streamer.reset();
|
||||
storage->close();
|
||||
|
||||
IndexStreamer::Pointer read_streamer =
|
||||
IndexFactory::CreateStreamer("FlatStreamer");
|
||||
ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
|
||||
auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
|
||||
ASSERT_NE(nullptr, read_storage);
|
||||
ASSERT_EQ(0, read_storage->init(stg_params));
|
||||
ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearchBuffer", false));
|
||||
ASSERT_EQ(0, read_streamer->open(read_storage));
|
||||
size_t topk = 3;
|
||||
auto provider = read_streamer->create_provider();
|
||||
for (size_t i = 0; i < cnt; i += 1) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ctx->set_topk(topk);
|
||||
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
|
||||
auto &result1 = ctx->result();
|
||||
ASSERT_EQ(topk, result1.size());
|
||||
IndexStorage::MemoryBlock block;
|
||||
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
|
||||
const float *data = (float *)block.data();
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
ASSERT_FLOAT_EQ(data[j], i);
|
||||
}
|
||||
ASSERT_EQ(i, result1[0].key());
|
||||
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i + 0.1f;
|
||||
}
|
||||
ctx->set_topk(topk);
|
||||
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
|
||||
auto &result2 = ctx->result();
|
||||
ASSERT_EQ(topk, result2.size());
|
||||
ASSERT_EQ(i, result2[0].key());
|
||||
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
|
||||
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
|
||||
}
|
||||
|
||||
ctx->set_topk(100U);
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = 10.1f;
|
||||
}
|
||||
ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
|
||||
auto &result = ctx->result();
|
||||
ASSERT_EQ(100U, result.size());
|
||||
ASSERT_EQ(10, result[0].key());
|
||||
ASSERT_EQ(11, result[1].key());
|
||||
ASSERT_EQ(5, result[10].key());
|
||||
ASSERT_EQ(0, result[20].key());
|
||||
ASSERT_EQ(30, result[30].key());
|
||||
ASSERT_EQ(35, result[35].key());
|
||||
ASSERT_EQ(99, result[99].key());
|
||||
|
||||
ElapsedTime elapsed_time;
|
||||
for (size_t i = 0; i < cnt; i += 1) {
|
||||
NumericalVector<float> vec(dim);
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i;
|
||||
}
|
||||
ctx->set_topk(topk);
|
||||
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
|
||||
auto &result1 = ctx->result();
|
||||
ASSERT_EQ(topk, result1.size());
|
||||
IndexStorage::MemoryBlock block;
|
||||
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
|
||||
const float *data = (float *)block.data();
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
ASSERT_FLOAT_EQ(data[j], i);
|
||||
}
|
||||
ASSERT_EQ(i, result1[0].key());
|
||||
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
vec[j] = i + 0.1f;
|
||||
}
|
||||
ctx->set_topk(topk);
|
||||
ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
|
||||
auto &result2 = ctx->result();
|
||||
ASSERT_EQ(topk, result2.size());
|
||||
ASSERT_EQ(i, result2[0].key());
|
||||
ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
|
||||
ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
|
||||
}
|
||||
cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
|
||||
|
||||
read_streamer->close();
|
||||
read_streamer.reset();
|
||||
}
|
||||
|
||||
|
||||
TEST_F(FlatStreamerTest, TestLinearSearchWithLRU) {
|
||||
MemoryLimitPool::get_instance().init(100 * 1024UL * 1024UL);
|
||||
#ifdef __ANDROID__
|
||||
@@ -351,7 +596,6 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) {
|
||||
ASSERT_EQ(topk, result1.size());
|
||||
IndexStorage::MemoryBlock block;
|
||||
ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
|
||||
const float *data = (float *)block.data();
|
||||
for (size_t j = 0; j < dim; ++j) {
|
||||
const float *data = (float *)provider->get_vector(result1[0].key());
|
||||
EXPECT_FLOAT_EQ(data[j], i);
|
||||
|
||||
Reference in New Issue
Block a user