Initial commit
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
|
||||
include(${CMAKE_SOURCE_DIR}/cmake/bazel.cmake)
|
||||
include(${CMAKE_SOURCE_DIR}/cmake/option.cmake)
|
||||
|
||||
if(APPLE)
|
||||
set(APPLE_FRAMEWORK_LIBS
|
||||
-framework CoreFoundation
|
||||
-framework CoreGraphics
|
||||
-framework CoreData
|
||||
-framework CoreText
|
||||
-framework Security
|
||||
-framework Foundation
|
||||
-Wl,-U,_MallocExtension_ReleaseFreeMemory
|
||||
-Wl,-U,_ProfilerStart
|
||||
-Wl,-U,_ProfilerStop
|
||||
-Wl,-U,_RegisterThriftProtocol
|
||||
)
|
||||
endif()
|
||||
|
||||
file(GLOB ALL_TEST_SRCS *_test.cc)
|
||||
foreach(CC_SRCS ${ALL_TEST_SRCS})
|
||||
get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
|
||||
cc_gmock(
|
||||
NAME ${CC_TARGET} STRICT
|
||||
LIBS zvec_common
|
||||
zvec_proto
|
||||
zvec_sqlengine
|
||||
zvec_ailego
|
||||
core_metric
|
||||
core_utility
|
||||
core_quantizer
|
||||
core_knn_hnsw core_knn_hnsw_sparse sparsehash
|
||||
core_knn_flat core_knn_flat_sparse core_knn_ivf
|
||||
core_mix_reducer
|
||||
${CMAKE_THREAD_LIBS_INIT}
|
||||
${CMAKE_DL_LIBS}
|
||||
SRCS ${CC_SRCS}
|
||||
INCS . ${CMAKE_SOURCE_DIR}/src ${CMAKE_SOURCE_DIR}/src/db ${CMAKE_SOURCE_DIR}/src/db/common
|
||||
LDFLAGS ${APPLE_FRAMEWORK_LIBS}
|
||||
)
|
||||
cc_test_suite(zvec_sqlengine ${CC_TARGET})
|
||||
endforeach()
|
||||
@@ -0,0 +1,484 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <memory>
|
||||
#include <arrow/api.h>
|
||||
#include <arrow/io/api.h>
|
||||
#include <arrow/ipc/api.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/common/file_helper.h"
|
||||
#include "db/index/segment/segment.h"
|
||||
#include "db/sqlengine/sqlengine.h"
|
||||
#include "index/common/index_params.h"
|
||||
#include "index/common/schema.h"
|
||||
#include "index/common/type.h"
|
||||
#include "test_helper.h"
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
static Doc create_doc(const uint64_t doc_id) {
|
||||
Doc new_doc;
|
||||
new_doc.set_pk("pk_" + std::to_string(doc_id));
|
||||
new_doc.set_doc_id(doc_id);
|
||||
|
||||
auto size = doc_id % 100;
|
||||
if (size > 0) {
|
||||
std::vector<std::string> str_array;
|
||||
std::vector<int32_t> i32_array;
|
||||
std::vector<int64_t> i64_array;
|
||||
std::vector<uint32_t> u32_array;
|
||||
std::vector<uint64_t> u64_array;
|
||||
std::vector<float> fp32_array;
|
||||
std::vector<double> fp64_array;
|
||||
std::vector<bool> bool_array;
|
||||
|
||||
for (uint32_t i = 1; i <= size; i++) {
|
||||
i32_array.push_back(i);
|
||||
i64_array.push_back(i);
|
||||
u32_array.push_back(i);
|
||||
u64_array.push_back(i);
|
||||
fp32_array.push_back(i);
|
||||
fp64_array.push_back(i);
|
||||
bool_array.push_back(i % 2 == 0);
|
||||
str_array.push_back("name" + std::to_string(i));
|
||||
}
|
||||
new_doc.set("i32_array", i32_array);
|
||||
new_doc.set("i64_array", i64_array);
|
||||
new_doc.set("u32_array", u32_array);
|
||||
new_doc.set("u64_array", u64_array);
|
||||
new_doc.set("fp32_array", fp32_array);
|
||||
new_doc.set("fp64_array", fp64_array);
|
||||
new_doc.set("bool_array", bool_array);
|
||||
new_doc.set("str_array", str_array);
|
||||
}
|
||||
return new_doc;
|
||||
}
|
||||
|
||||
class ContainTest : public testing::Test {
|
||||
protected:
|
||||
static void SetUpTestSuite() {
|
||||
FileHelper::RemoveDirectory(seg_path_);
|
||||
FileHelper::CreateDirectory(seg_path_);
|
||||
|
||||
auto invert_params = std::make_shared<InvertIndexParams>(true);
|
||||
collection_schema_ = std::make_shared<CollectionSchema>(
|
||||
"test_collection",
|
||||
std::vector<FieldSchema::Ptr>{
|
||||
std::make_shared<FieldSchema>("str_array", DataType::ARRAY_STRING,
|
||||
true, nullptr),
|
||||
std::make_shared<FieldSchema>("i32_array", DataType::ARRAY_INT32,
|
||||
true, nullptr),
|
||||
std::make_shared<FieldSchema>("i64_array", DataType::ARRAY_INT64,
|
||||
true, nullptr),
|
||||
std::make_shared<FieldSchema>("u32_array", DataType::ARRAY_UINT32,
|
||||
true, nullptr),
|
||||
std::make_shared<FieldSchema>("u64_array", DataType::ARRAY_UINT64,
|
||||
true, nullptr),
|
||||
std::make_shared<FieldSchema>("fp32_array", DataType::ARRAY_FLOAT,
|
||||
true, nullptr),
|
||||
std::make_shared<FieldSchema>("fp64_array", DataType::ARRAY_DOUBLE,
|
||||
true, nullptr),
|
||||
std::make_shared<FieldSchema>("bool_array", DataType::ARRAY_BOOL,
|
||||
true, nullptr),
|
||||
|
||||
});
|
||||
|
||||
auto segment = create_segment(seg_path_, *collection_schema_);
|
||||
if (segment == nullptr) {
|
||||
LOG_ERROR("create segment failed");
|
||||
EXPECT_TRUE(segment != nullptr);
|
||||
std::exit(EXIT_FAILURE);
|
||||
}
|
||||
auto status = InsertDoc(segment, 0, 10000, &create_doc);
|
||||
if (!status.ok()) {
|
||||
LOG_ERROR("insert doc failed: %s", status.c_str());
|
||||
EXPECT_TRUE(status.ok());
|
||||
std::exit(EXIT_FAILURE);
|
||||
}
|
||||
segments_.push_back(segment);
|
||||
}
|
||||
|
||||
static void TearDownTestSuite() {
|
||||
segments_.clear();
|
||||
FileHelper::RemoveDirectory(seg_path_);
|
||||
}
|
||||
|
||||
protected:
|
||||
static inline std::string seg_path_ = "./test_collection";
|
||||
static inline CollectionSchema::Ptr collection_schema_;
|
||||
static inline std::vector<Segment::Ptr> segments_;
|
||||
};
|
||||
|
||||
|
||||
TEST_F(ContainTest, ContainAllInt32) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "i32_array contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAllInt64) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "i64_array contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAllUint32) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "u32_array contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAllUint64) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "u64_array contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAllFp32) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "fp32_array contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAllFp64) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "fp64_array contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAllString) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "str_array contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += "'name" + std::to_string(i) + "'";
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAnyInt32) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "i32_array contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 98; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 98) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAnyInt64) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "i64_array contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 98; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 98) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAnyUint32) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "u32_array contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 98; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 98) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAnyUint64) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "u64_array contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 98; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 98) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAnyFp32) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "fp32_array contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 98; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 98) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAnyFp64) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "fp64_array contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 98; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 98) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ContainTest, ContainAnyString) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "str_array contain_any ('name98','name99','name100')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 98; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 98) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
@@ -0,0 +1,938 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/sqlengine/sqlengine.h"
|
||||
#include "index/common/schema.h"
|
||||
#include "recall_base.h"
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
class ForwardRecallTest : public RecallTest {};
|
||||
|
||||
TEST_F(ForwardRecallTest, Basic) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int i = 0; i < query.topk_; i++) {
|
||||
auto &doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, BasicWithDocId) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.include_doc_id_ = true;
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int i = 0; i < query.topk_; i++) {
|
||||
auto &doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(doc->doc_id(), i);
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, OutputNoFields) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>{};
|
||||
query.topk_ = 200;
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int i = 0; i < query.topk_; i++) {
|
||||
auto &doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(doc->field_names().size(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, DenseVector) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "dense"};
|
||||
query.topk_ = 200;
|
||||
query.include_vector_ = true;
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int i = 0; i < query.topk_; i++) {
|
||||
auto &doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto dense = doc->get<std::vector<float>>("dense");
|
||||
ASSERT_TRUE(dense.has_value());
|
||||
EXPECT_EQ(dense.value().size(), 4);
|
||||
for (auto v : dense.value()) {
|
||||
EXPECT_FLOAT_EQ(v, (float)i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, SparseVector) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "sparse"};
|
||||
query.topk_ = 200;
|
||||
query.include_vector_ = true;
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int i = 0; i < query.topk_; i++) {
|
||||
auto &doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
// EXPECT_EQ(doc->field_names().size(), 2);
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto sparse =
|
||||
doc->get<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
||||
"sparse");
|
||||
if (i % 100 == 0) {
|
||||
// set with empty vector
|
||||
ASSERT_FALSE(sparse.has_value());
|
||||
continue;
|
||||
}
|
||||
|
||||
ASSERT_TRUE(sparse.has_value());
|
||||
const auto &[indices, values] = sparse.value();
|
||||
EXPECT_EQ(indices.size(), i % 100);
|
||||
EXPECT_EQ(values.size(), i % 100);
|
||||
for (int j = 0; j < i % 100; j++) {
|
||||
EXPECT_EQ(indices[j], j);
|
||||
EXPECT_FLOAT_EQ(values[j], (float)i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, MultiSegment) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>();
|
||||
query.topk_ = 200;
|
||||
query.include_vector_ = true;
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
std::vector<Segment::Ptr> segments = segments_;
|
||||
segments.push_back(segments_[0]);
|
||||
auto ret = engine->execute(collection_schema_, query, segments);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int i = 0; i < query.topk_; i++) {
|
||||
auto &doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto dense = doc->get<std::vector<float>>("dense");
|
||||
ASSERT_TRUE(dense.has_value());
|
||||
EXPECT_EQ(dense.value().size(), 4);
|
||||
for (auto v : dense.value()) {
|
||||
EXPECT_FLOAT_EQ(v, (float)i);
|
||||
}
|
||||
|
||||
auto sparse =
|
||||
doc->get<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
||||
"sparse");
|
||||
if (i % 100 == 0) {
|
||||
// set with empty vector
|
||||
ASSERT_FALSE(sparse.has_value());
|
||||
continue;
|
||||
}
|
||||
|
||||
ASSERT_TRUE(sparse.has_value());
|
||||
const auto &[indices, values] = sparse.value();
|
||||
EXPECT_EQ(indices.size(), i % 100);
|
||||
EXPECT_EQ(values.size(), i % 100);
|
||||
for (int j = 0; j < i % 100; j++) {
|
||||
EXPECT_EQ(indices[j], j);
|
||||
EXPECT_FLOAT_EQ(values[j], (float)i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, Eq) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "age = 1";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++, i += 100) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, Gt) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "id > 1000";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int j = 0; j < query.topk_; j++) {
|
||||
auto &doc = docs[j];
|
||||
auto i = j + 1001;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, Ge) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "id >= 1000";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int j = 0; j < query.topk_; j++) {
|
||||
auto &doc = docs[j];
|
||||
auto i = j + 1000;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, Lt) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "id < 100";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 1) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, Le) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "id <= 100";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 101);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 1) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, And) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "id <= 100 and id > 50";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 50);
|
||||
for (int j = 0, i = 51; j < (int)docs.size(); j++, i += 1) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, Or) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "id < 100 or id > 200";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 200);
|
||||
for (int j = 0; j < (int)docs.size(); j++) {
|
||||
int i = j < 100 ? j : j + 101;
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, StrEq) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name = 'user_1'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++, i += 100) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, StrGe) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name >= 'user_1'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 1) {
|
||||
if (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, StrIn) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name IN ('user_1', 'user_2')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
if (i % 100 == 1) {
|
||||
i += 1;
|
||||
} else if (i % 100 == 2) {
|
||||
i += 99;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, StrNotIn) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name NOT IN ('user_1', 'user_2')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
if (i % 100 == 0) {
|
||||
i += 3;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, StrLike) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name like 'user_9%'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 9; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
if (i % 100 == 9) {
|
||||
i += 81;
|
||||
} else if (i % 100 == 99) {
|
||||
i += 10;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, IsNull) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "optional_age is null";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 100) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, IsNotNull) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "optional_age is not null";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 1) {
|
||||
if (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, IsNullNoResult) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "age is null";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 0);
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, ContainAll) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "category_set contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, NotContainAll) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "category_set not contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
// i % 100 == 0 has null category
|
||||
while (i % 100 >= 32 || i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, ContainAny) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "category_set contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 98; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 98) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, NotContainAny) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "category_set not contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
// i % 100 == 0 has null category
|
||||
while (i % 100 >= 98 || i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, BoolContainAll) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "bool_array contain_all (true, false)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 3;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, BoolContainAny) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "bool_array contain_any (true)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
if (i % 3 == 2) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, ContainAllEmptySet) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "category_set contain_all ()";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, NotContainAllEmptySet) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "category_set not contain_all ()";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 0);
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, ContainAnyEmptySet) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "category_set contain_any ()";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 0);
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, NotContainAnyEmptySet) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "category_set not contain_any ()";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, BoolEqTrue) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "bool = TRuE";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 100) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, BoolEqFalse) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "bool = false";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
if (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, ArrayLengthEq) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "array_length(category_set) = 32";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 100;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ForwardRecallTest, ArrayLengthGe) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "array_length(category_set) >= 32";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
@@ -0,0 +1,789 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/sqlengine/sqlengine.h"
|
||||
#include "index/common/schema.h"
|
||||
#include "recall_base.h"
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
class InvertRecallTest : public RecallTest {};
|
||||
|
||||
TEST_F(InvertRecallTest, Eq) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_age = 1";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++, i += 100) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, Gt) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_id > 1000";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int j = 0; j < query.topk_; j++) {
|
||||
auto &doc = docs[j];
|
||||
auto i = j + 1001;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, Ge) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_id >= 1000";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int j = 0; j < query.topk_; j++) {
|
||||
auto &doc = docs[j];
|
||||
auto i = j + 1000;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, Lt) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_id < 100";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 1) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, Le) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_id <= 100";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 101);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 1) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, And) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_id <= 100 and invert_id > 50";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 50);
|
||||
for (int j = 0, i = 51; j < (int)docs.size(); j++, i += 1) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, Or) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_id < 100 or invert_id > 200";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 200);
|
||||
for (int j = 0; j < (int)docs.size(); j++) {
|
||||
int i = j < 100 ? j : j + 101;
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, StrEq) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_name = 'user_1'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++, i += 100) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, StrGe) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_name >= 'user_1'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 1) {
|
||||
if (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, StrIn) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_name IN ('user_1', 'user_2')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
if (i % 100 == 1) {
|
||||
i += 1;
|
||||
} else if (i % 100 == 2) {
|
||||
i += 99;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, StrNotIn) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_name NOT IN ('user_1', 'user_2')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
if (i % 100 == 0) {
|
||||
i += 3;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, StrLike) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_name like 'user\\_9%'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 9; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
if (i % 100 == 9) {
|
||||
i += 81;
|
||||
} else if (i % 100 == 99) {
|
||||
i += 10;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, ContainAll) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_category_set contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, NotContainAll) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_category_set not contain_all (";
|
||||
for (int i = 1; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
// i % 100 == 0 has null category
|
||||
while (i % 100 >= 32 || i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, ContainAny) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_category_set contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 98; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 98) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, NotContainAny) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_category_set not contain_any (98,99,100)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
// i % 100 == 0 has null category
|
||||
while (i % 100 >= 98 || i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, BoolContainAll) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_bool_array contain_all (true, false)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 3;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, BoolContainAny) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_bool_array contain_any (true)";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
if (i % 3 == 2) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, ContainAllEmptySet) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_category_set contain_all ()";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, NotContainAllEmptySet) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_category_set not contain_all ()";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 0);
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, ContainAnyEmptySet) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_category_set contain_any ()";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
ASSERT_EQ(docs.size(), 0);
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, NotContainAnyEmptySet) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_category_set not contain_any ()";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, IsNull) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_optional_age is null";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 100) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, IsNotNull) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_optional_age is not null";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 1) {
|
||||
if (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, BoolEqTrue) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_bool = TRuE";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 0; j < (int)docs.size(); j++, i += 100) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, BoolEqFalse) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_bool = false";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 1; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
if (i % 100 == 0) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, ArrayLengthGe) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "array_length(invert_category_set) >= 32";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 200);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 1;
|
||||
while (i % 100 < 32) {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, ArrayLengthEq) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "array_length(invert_category_set) = 32";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (int j = 0, i = 32; j < (int)docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
EXPECT_EQ(i, doc->get<uint64_t>("id"));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
|
||||
i += 100;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InvertRecallTest, MultiSegment) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = std::vector<std::string>();
|
||||
query.topk_ = 200;
|
||||
query.include_vector_ = true;
|
||||
query.filter_ = "invert_id <= 5000";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
std::vector<Segment::Ptr> segments = segments_;
|
||||
segments.push_back(segments_[0]);
|
||||
auto ret = engine->execute(collection_schema_, query, segments);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int i = 0; i < query.topk_; i++) {
|
||||
auto &doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto dense = doc->get<std::vector<float>>("dense");
|
||||
ASSERT_TRUE(dense.has_value());
|
||||
EXPECT_EQ(dense.value().size(), 4);
|
||||
for (auto v : dense.value()) {
|
||||
EXPECT_FLOAT_EQ(v, (float)i);
|
||||
}
|
||||
|
||||
auto sparse =
|
||||
doc->get<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
||||
"sparse");
|
||||
if (i % 100 == 0) {
|
||||
// set with empty vector
|
||||
ASSERT_FALSE(sparse.has_value());
|
||||
continue;
|
||||
}
|
||||
|
||||
ASSERT_TRUE(sparse.has_value());
|
||||
const auto &[indices, values] = sparse.value();
|
||||
EXPECT_EQ(indices.size(), i % 100);
|
||||
EXPECT_EQ(values.size(), i % 100);
|
||||
for (int j = 0; j < i % 100; j++) {
|
||||
EXPECT_EQ(indices[j], j);
|
||||
EXPECT_FLOAT_EQ(values[j], (float)i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
@@ -0,0 +1,372 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <arrow/api.h>
|
||||
#include <arrow/io/api.h>
|
||||
#include <arrow/ipc/api.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/common/file_helper.h"
|
||||
#include "db/index/common/version_manager.h"
|
||||
#include "db/index/segment/segment.h"
|
||||
#include "db/sqlengine/sqlengine.h"
|
||||
#include "index/common/index_params.h"
|
||||
#include "index/common/schema.h"
|
||||
#include "index/common/type.h"
|
||||
#include "test_helper.h"
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
static Doc create_doc(const uint64_t doc_id) {
|
||||
Doc new_doc;
|
||||
new_doc.set_pk("pk_" + std::to_string(doc_id));
|
||||
new_doc.set_doc_id(doc_id);
|
||||
|
||||
auto name = std::string("user-");
|
||||
if (doc_id >= 5000 && doc_id < 8000) {
|
||||
name += "%";
|
||||
} else if (doc_id >= 8000) {
|
||||
name += '_';
|
||||
}
|
||||
name += std::to_string(doc_id % 100);
|
||||
new_doc.set<std::string>("name", name);
|
||||
new_doc.set<std::string>("invert_name", name);
|
||||
new_doc.set<std::string>("extended_invert_name", name);
|
||||
return new_doc;
|
||||
}
|
||||
|
||||
class LikeTest : public testing::Test {
|
||||
protected:
|
||||
static void SetUpTestSuite() {
|
||||
FileHelper::RemoveDirectory(seg_path_);
|
||||
FileHelper::CreateDirectory(seg_path_);
|
||||
|
||||
auto invert_params = std::make_shared<InvertIndexParams>(true);
|
||||
collection_schema_ = std::make_shared<CollectionSchema>(
|
||||
"test_collection",
|
||||
std::vector<FieldSchema::Ptr>{
|
||||
std::make_shared<FieldSchema>("name", DataType::STRING, false,
|
||||
nullptr),
|
||||
std::make_shared<FieldSchema>(
|
||||
"invert_name", DataType::STRING, false,
|
||||
std::make_shared<InvertIndexParams>(false, false)),
|
||||
std::make_shared<FieldSchema>(
|
||||
"extended_invert_name", DataType::STRING, false,
|
||||
std::make_shared<InvertIndexParams>(false, true)),
|
||||
});
|
||||
auto segment = create_segment(seg_path_, *collection_schema_);
|
||||
if (segment == nullptr) {
|
||||
LOG_ERROR("create segment failed");
|
||||
EXPECT_TRUE(segment != nullptr);
|
||||
std::exit(EXIT_FAILURE);
|
||||
}
|
||||
auto status = InsertDoc(segment, 0, 10000, &create_doc);
|
||||
if (!status.ok()) {
|
||||
LOG_ERROR("insert doc failed: %s", status.c_str());
|
||||
EXPECT_TRUE(status.ok());
|
||||
std::exit(EXIT_FAILURE);
|
||||
}
|
||||
segments_.push_back(segment);
|
||||
}
|
||||
|
||||
static void TearDownTestSuite() {
|
||||
segments_.clear();
|
||||
FileHelper::RemoveDirectory(seg_path_);
|
||||
}
|
||||
|
||||
protected:
|
||||
static inline std::string seg_path_ = "./test_collection";
|
||||
static inline CollectionSchema::Ptr collection_schema_;
|
||||
static inline std::vector<Segment::Ptr> segments_;
|
||||
};
|
||||
|
||||
|
||||
TEST_F(LikeTest, ForwardLikeAll) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name like '%'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
auto doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, InvertLikeAll) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_name like '%'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
auto doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, ForwardPrefixLike) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name like 'user-22%'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
auto doc = docs[i];
|
||||
int doc_id = i * 100 + 22;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, InvertPrefixLike) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_name like 'user-22%'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
auto doc = docs[i];
|
||||
int doc_id = i * 100 + 22;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, ForwardSuffixLike) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name like '%ser-22'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
auto doc = docs[i];
|
||||
int doc_id = i * 100 + 22;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, NotExtendedInvertSuffixLikeRunAsForward) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_name like '%ser-22'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
auto doc = docs[i];
|
||||
int doc_id = i * 100 + 22;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, ExtendedInvertSuffixLike) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "extended_invert_name like '%ser-22'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
auto doc = docs[i];
|
||||
int doc_id = i * 100 + 22;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, ForwardMiddleLike) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name like 'user%2'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0, doc_id = 0; i < docs.size(); i++, doc_id++) {
|
||||
auto doc = docs[i];
|
||||
while (doc_id % 100 % 10 != 2) {
|
||||
doc_id++;
|
||||
}
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, ExtendedInvertMiddleLike) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "extended_invert_name like 'user%2'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0, doc_id = 0; i < docs.size(); i++, doc_id++) {
|
||||
auto doc = docs[i];
|
||||
while (doc_id % 100 % 10 != 2) {
|
||||
doc_id++;
|
||||
}
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, UnderScore) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name like 'user-_2'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0, doc_id = 0; i < docs.size(); i++, doc_id++) {
|
||||
auto doc = docs[i];
|
||||
while (doc_id % 100 % 10 != 2 || doc_id % 100 < 10) {
|
||||
doc_id++;
|
||||
}
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, InvertUnderScoreRunAsForward) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "invert_name like 'user-_2'";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0, doc_id = 0; i < docs.size(); i++, doc_id++) {
|
||||
auto doc = docs[i];
|
||||
while (doc_id % 100 % 10 != 2 || doc_id % 100 < 10) {
|
||||
doc_id++;
|
||||
}
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, ForwardEscapePercent) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = R"(name like 'user-\%%')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0, doc_id = 5000; i < docs.size(); i++, doc_id++) {
|
||||
auto doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, InvertEscapePercent) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = R"(invert_name like 'user-\%%')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0, doc_id = 5000; i < docs.size(); i++, doc_id++) {
|
||||
auto doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, ForwardEscapeUnderscore) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = R"(name like 'user-\_%')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0, doc_id = 8000; i < docs.size(); i++, doc_id++) {
|
||||
auto doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, InvertEscapeUnderscore) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = R"(invert_name like 'user-\_%')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0, doc_id = 8000; i < docs.size(); i++, doc_id++) {
|
||||
auto doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LikeTest, NoPercentRunAsEqual) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name"};
|
||||
query.topk_ = 200;
|
||||
query.filter_ = R"(invert_name like 'user-22')";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
EXPECT_TRUE(ret.has_value()) << ret.error();
|
||||
auto docs = std::move(ret.value());
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
auto doc = docs[i];
|
||||
int doc_id = i * 100 + 22;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
@@ -0,0 +1,513 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <ailego/logger/logger.h>
|
||||
#include <arrow/api.h>
|
||||
#include <arrow/compute/api.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/index/column/common/index_results.h"
|
||||
#include "db/index/column/vector_column/vector_column_indexer.h"
|
||||
#include "db/index/segment/segment.h"
|
||||
#include "index/column/inverted_column/inverted_column_indexer.h"
|
||||
#include "index/column/vector_column/vector_column_params.h"
|
||||
#include "index/common/index_filter.h"
|
||||
namespace zvec {
|
||||
|
||||
|
||||
class MockIndexResult : public InvertedSearchResult {
|
||||
public:
|
||||
MockIndexResult(const std::vector<idx_t> &doc_ids,
|
||||
const std::vector<float> &scores)
|
||||
: doc_ids_(doc_ids), scores_(scores) {}
|
||||
|
||||
MockIndexResult(const std::vector<idx_t> &doc_ids,
|
||||
const std::vector<float> &scores,
|
||||
const std::vector<std::string> &groups)
|
||||
: doc_ids_(doc_ids), scores_(scores), group_ids_(groups) {}
|
||||
|
||||
size_t count() const override {
|
||||
return doc_ids_.size();
|
||||
}
|
||||
|
||||
IteratorUPtr create_iterator() override {
|
||||
return std::make_unique<MockIterator>(*this);
|
||||
}
|
||||
|
||||
private:
|
||||
struct MockIterator : public IndexResults::Iterator {
|
||||
MockIterator(MockIndexResult &parent) : parent_(parent) {}
|
||||
|
||||
idx_t doc_id() const override {
|
||||
return parent_.doc_ids_[current_index_];
|
||||
}
|
||||
|
||||
float score() const override {
|
||||
return parent_.scores_[current_index_];
|
||||
}
|
||||
|
||||
void next() override {
|
||||
++current_index_;
|
||||
}
|
||||
|
||||
bool valid() const override {
|
||||
return current_index_ < parent_.count();
|
||||
}
|
||||
|
||||
const std::string &group_id() const override {
|
||||
return parent_.group_ids_[current_index_];
|
||||
}
|
||||
|
||||
MockIndexResult &parent_;
|
||||
size_t current_index_{0};
|
||||
};
|
||||
|
||||
std::vector<idx_t> doc_ids_;
|
||||
std::vector<float> scores_;
|
||||
std::vector<std::string> group_ids_;
|
||||
};
|
||||
|
||||
class MockVectorIndexer : public CombinedVectorColumnIndexer {
|
||||
public:
|
||||
//! Search results with query
|
||||
Result<IndexResults::Ptr> Search(
|
||||
const vector_column_params::VectorData &vector_data,
|
||||
const vector_column_params::QueryParams &query_params) override {
|
||||
// return tl::make_unexpected(Status::InternalError("err"));
|
||||
return std::make_shared<MockIndexResult>(
|
||||
std::vector<idx_t>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
||||
std::vector<float>{0.1F, 0.2F, 0.3F, 0.4F, 0.5F, 0.6F, 0.7F, 0.8F, 0.9F,
|
||||
1.0F},
|
||||
std::vector<std::string>{"group_0", "group_1", "group_2", "group_0",
|
||||
"group_1", "group_2", "group_0", "group_1",
|
||||
"group_2", "group_0"});
|
||||
}
|
||||
|
||||
Result<vector_column_params::VectorDataBuffer> Fetch(
|
||||
uint32_t doc_id) const override {
|
||||
// float f = doc_id;
|
||||
// std::vector<float> v(4, f);
|
||||
// std::string v_str = std::string(reinterpret_cast<char *>(v.data()),
|
||||
// v.size() * sizeof(float));
|
||||
// return vector_column_params::VectorDataBuffer{
|
||||
// vector_column_params::DenseVectorBuffer{v_str}};
|
||||
|
||||
// sparse
|
||||
uint32_t count = doc_id % 5;
|
||||
std::vector<uint32_t> indices(count);
|
||||
std::vector<float> values(count);
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
indices[i] = i;
|
||||
values[i] = i / 100.0;
|
||||
}
|
||||
return vector_column_params::VectorDataBuffer{
|
||||
vector_column_params::SparseVectorBuffer{
|
||||
std::string(reinterpret_cast<char *>(indices.data()),
|
||||
indices.size() * sizeof(uint32_t)),
|
||||
std::string(reinterpret_cast<char *>(values.data()),
|
||||
values.size() * sizeof(float))}};
|
||||
}
|
||||
};
|
||||
|
||||
class MockInvertIndexer : public InvertedColumnIndexer {
|
||||
public:
|
||||
MockInvertIndexer() : InvertedColumnIndexer(ctx) {}
|
||||
|
||||
InvertedSearchResult::Ptr search(const std::string &value,
|
||||
CompareOp op) const override {
|
||||
return std::make_shared<MockIndexResult>(
|
||||
std::vector<idx_t>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
||||
std::vector<float>{0.1F, 0.2F, 0.3F, 0.4F, 0.5F, 0.6F, 0.7F, 0.8F, 0.9F,
|
||||
1.0F});
|
||||
}
|
||||
|
||||
InvertedSearchResult::Ptr search_null() const override {
|
||||
return std::make_shared<MockIndexResult>(
|
||||
std::vector<idx_t>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
||||
std::vector<float>{0.1F, 0.2F, 0.3F, 0.4F, 0.5F, 0.6F, 0.7F, 0.8F, 0.9F,
|
||||
1.0F});
|
||||
}
|
||||
|
||||
InvertedSearchResult::Ptr search_non_null() const override {
|
||||
return std::make_shared<MockIndexResult>(
|
||||
std::vector<idx_t>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
||||
std::vector<float>{0.1F, 0.2F, 0.3F, 0.4F, 0.5F, 0.6F, 0.7F, 0.8F, 0.9F,
|
||||
1.0F});
|
||||
}
|
||||
|
||||
private:
|
||||
RocksdbContext ctx;
|
||||
};
|
||||
|
||||
// std::make_shared<FieldSchema>("id", DataType::INT32, false, 0, false,
|
||||
// nullptr),
|
||||
// std::make_shared<FieldSchema>("name", DataType::STRING, false, 0,
|
||||
// false, nullptr),
|
||||
// std::make_shared<FieldSchema>("age", DataType::INT64, false, 0,
|
||||
// false, nullptr),
|
||||
// std::make_shared<FieldSchema>("score", DataType::DOUBLE, false, 0,
|
||||
// false, nullptr),
|
||||
inline arrow::Result<std::shared_ptr<arrow::Table>> CreateTable(
|
||||
int count = 10000000) {
|
||||
auto schema = arrow::schema({
|
||||
arrow::field("id", arrow::int32()),
|
||||
arrow::field("name", arrow::utf8()),
|
||||
arrow::field("age", arrow::int64()),
|
||||
arrow::field("score", arrow::float64()),
|
||||
arrow::field("_zvec_uid_", arrow::utf8()),
|
||||
arrow::field("_zvec_row_id_", arrow::uint64()),
|
||||
arrow::field("_zvec_g_doc_id_", arrow::uint64()),
|
||||
arrow::field("tag_list", arrow::list(arrow::int32())),
|
||||
});
|
||||
std::shared_ptr<arrow::Array> array_id;
|
||||
std::shared_ptr<arrow::Array> array_name;
|
||||
std::shared_ptr<arrow::Array> array_age;
|
||||
std::shared_ptr<arrow::Array> array_score;
|
||||
std::shared_ptr<arrow::Array> array_uid;
|
||||
arrow::NumericBuilder<arrow::Int64Type> builder;
|
||||
auto has_value = [](int i) { return i % 13 != 0; };
|
||||
ARROW_RETURN_NOT_OK(builder.Reserve(count));
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (has_value(i)) {
|
||||
ARROW_RETURN_NOT_OK((builder.Append(i)));
|
||||
} else {
|
||||
ARROW_RETURN_NOT_OK((builder.AppendNull()));
|
||||
}
|
||||
}
|
||||
ARROW_RETURN_NOT_OK(builder.Finish(&array_age));
|
||||
builder.Reset();
|
||||
|
||||
arrow::NumericBuilder<arrow::Int32Type> builder_id;
|
||||
ARROW_RETURN_NOT_OK(builder_id.Reserve(count));
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (has_value(i)) {
|
||||
ARROW_RETURN_NOT_OK((builder_id.Append(i)));
|
||||
} else {
|
||||
ARROW_RETURN_NOT_OK((builder_id.AppendNull()));
|
||||
}
|
||||
}
|
||||
ARROW_RETURN_NOT_OK(builder_id.Finish(&array_id));
|
||||
|
||||
arrow::NumericBuilder<arrow::DoubleType> builder_score;
|
||||
ARROW_RETURN_NOT_OK(builder_score.Reserve(count));
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (has_value(i)) {
|
||||
ARROW_RETURN_NOT_OK((builder_score.Append(i / 100.0)));
|
||||
} else {
|
||||
ARROW_RETURN_NOT_OK((builder_score.AppendNull()));
|
||||
}
|
||||
}
|
||||
ARROW_RETURN_NOT_OK(builder_score.Finish(&array_score));
|
||||
|
||||
|
||||
arrow::StringBuilder builder_d;
|
||||
ARROW_RETURN_NOT_OK(builder_d.Reserve(count));
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (has_value(i)) {
|
||||
ARROW_RETURN_NOT_OK((builder_d.Append("name_" + std::to_string(i))));
|
||||
} else {
|
||||
ARROW_RETURN_NOT_OK((builder_d.AppendNull()));
|
||||
}
|
||||
}
|
||||
ARROW_RETURN_NOT_OK(builder_d.Finish(&array_name));
|
||||
|
||||
arrow::StringBuilder builder_uid;
|
||||
ARROW_RETURN_NOT_OK(builder_uid.Reserve(count));
|
||||
for (int i = 0; i < count; i++) {
|
||||
ARROW_RETURN_NOT_OK((builder_uid.Append("uid_" + std::to_string(i))));
|
||||
}
|
||||
ARROW_RETURN_NOT_OK(builder_uid.Finish(&array_uid));
|
||||
|
||||
arrow::NumericBuilder<arrow::UInt64Type> builder_row_id;
|
||||
ARROW_RETURN_NOT_OK(builder_row_id.Reserve(count));
|
||||
for (int i = 0; i < count; i++) {
|
||||
ARROW_RETURN_NOT_OK((builder_row_id.Append(i)));
|
||||
}
|
||||
std::shared_ptr<arrow::Array> array_row_id;
|
||||
ARROW_RETURN_NOT_OK(builder_row_id.Finish(&array_row_id));
|
||||
|
||||
arrow::NumericBuilder<arrow::UInt64Type> builder_doc_id;
|
||||
ARROW_RETURN_NOT_OK(builder_doc_id.Reserve(count));
|
||||
for (int i = 0; i < count; i++) {
|
||||
ARROW_RETURN_NOT_OK((builder_doc_id.Append(i)));
|
||||
}
|
||||
std::shared_ptr<arrow::Array> array_doc_id;
|
||||
ARROW_RETURN_NOT_OK(builder_doc_id.Finish(&array_doc_id));
|
||||
|
||||
arrow::ListBuilder list_builder(arrow::default_memory_pool(),
|
||||
std::make_shared<arrow::Int32Builder>());
|
||||
auto *tag_value_builder =
|
||||
static_cast<arrow::Int32Builder *>(list_builder.value_builder());
|
||||
|
||||
for (int i = 0; i < count; ++i) {
|
||||
// 开始一个新的 list
|
||||
ARROW_RETURN_NOT_OK(list_builder.Append());
|
||||
|
||||
int idx = i % 5; // 对应模式
|
||||
for (int j = 0; j < idx + 1; ++j) {
|
||||
ARROW_RETURN_NOT_OK(tag_value_builder->Append(j + 1));
|
||||
}
|
||||
}
|
||||
std::shared_ptr<arrow::Array> tag_list_array;
|
||||
auto status = list_builder.Finish(&tag_list_array);
|
||||
;
|
||||
|
||||
return arrow::Table::Make(
|
||||
schema, {array_id, array_name, array_age, array_score, array_uid,
|
||||
array_row_id, array_doc_id, tag_list_array});
|
||||
}
|
||||
|
||||
class MockIndexFilter : public IndexFilter {
|
||||
public:
|
||||
bool is_filtered(uint64_t id) const override {
|
||||
return id % 2 == 1;
|
||||
}
|
||||
};
|
||||
|
||||
inline arrow::Result<std::shared_ptr<Table>> TakeRowsByIndices(
|
||||
const std::shared_ptr<Table> &table, const std::vector<int> &row_indices) {
|
||||
arrow::MemoryPool *pool = arrow::default_memory_pool();
|
||||
arrow::Int32Builder indices_builder(pool);
|
||||
ARROW_RETURN_NOT_OK(
|
||||
indices_builder.AppendValues(row_indices.data(), row_indices.size()));
|
||||
std::shared_ptr<arrow::Array> indices_array;
|
||||
ARROW_RETURN_NOT_OK(indices_builder.Finish(&indices_array));
|
||||
|
||||
|
||||
// 2. 对每一列执行 Take 操作
|
||||
std::vector<std::shared_ptr<arrow::ChunkedArray>> new_columns;
|
||||
for (const auto &column : table->columns()) {
|
||||
// 使用 Take 提取指定索引的元素
|
||||
ARROW_ASSIGN_OR_RAISE(auto taken_array, cp::Take(column, indices_array));
|
||||
new_columns.emplace_back(taken_array.chunked_array());
|
||||
}
|
||||
|
||||
// 3. 构造新的 Table
|
||||
return arrow::Table::Make(table->schema(), new_columns, row_indices.size());
|
||||
}
|
||||
|
||||
|
||||
class MockSegment : public Segment {
|
||||
public:
|
||||
MockSegment() : Segment() {}
|
||||
|
||||
virtual ~MockSegment() = default;
|
||||
|
||||
SegmentID id() const override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
TablePtr fetch(const std::vector<std::string> &columns,
|
||||
const std::vector<int> &indices) const override {
|
||||
std::string s = "";
|
||||
for (auto i : indices) {
|
||||
s += std::to_string(i);
|
||||
s += ",";
|
||||
}
|
||||
LOG_INFO("Fetch indices: %s %s", get_column_names(columns).c_str(),
|
||||
s.c_str());
|
||||
auto table = CreateTable(1000).MoveValueUnsafe();
|
||||
|
||||
auto res = TakeRowsByIndices(table, indices);
|
||||
if (!res.ok()) {
|
||||
LOG_ERROR("Take error: %s", res.status().ToString().c_str());
|
||||
return nullptr;
|
||||
}
|
||||
LOG_INFO("Take: %s", res.ValueOrDie()->ToString().c_str());
|
||||
return res.MoveValueUnsafe();
|
||||
}
|
||||
|
||||
ExecBatchPtr fetch(const std::vector<std::string> &columns,
|
||||
int index) const override {
|
||||
LOG_ERROR("Not implemented");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
static std::string get_column_names(const std::vector<std::string> &columns) {
|
||||
std::string s = "";
|
||||
for (auto i : columns) {
|
||||
s += i;
|
||||
s += ",";
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
RecordBatchReaderPtr scan(
|
||||
const std::vector<std::string> &columns) const override {
|
||||
auto table = CreateTable(10000);
|
||||
LOG_INFO("Scan return: %s %s", get_column_names(columns).c_str(),
|
||||
table.ValueOrDie()->ToString().c_str());
|
||||
return std::make_shared<arrow::TableBatchReader>(table.ValueOrDie());
|
||||
}
|
||||
|
||||
const IndexFilter::Ptr get_filter() override {
|
||||
return std::make_shared<MockIndexFilter>();
|
||||
}
|
||||
|
||||
CombinedVectorColumnIndexer::Ptr get_quant_combined_vector_indexer(
|
||||
const std::string &field_name) const override {
|
||||
return std::make_shared<MockVectorIndexer>();
|
||||
}
|
||||
|
||||
CombinedVectorColumnIndexer::Ptr get_combined_vector_indexer(
|
||||
const std::string &field_name) const override {
|
||||
return std::make_shared<MockVectorIndexer>();
|
||||
}
|
||||
|
||||
InvertedColumnIndexer::Ptr get_scalar_indexer(
|
||||
const std::string &field_name) const override {
|
||||
return std::make_shared<MockInvertIndexer>();
|
||||
}
|
||||
|
||||
SegmentMeta::Ptr meta() const override {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint64_t doc_count(const IndexFilter::Ptr filter = nullptr) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
Status add_column(const std::string &column_name,
|
||||
FieldSchema::Ptr column_schema,
|
||||
const std::string &expression,
|
||||
const AddColumnOptions &options) override {
|
||||
return Status::InternalError();
|
||||
}
|
||||
|
||||
Status alter_column(const std::string &column_name,
|
||||
const FieldSchema::Ptr &new_column_schema,
|
||||
const AlterColumnOptions &options) override {
|
||||
return Status::InternalError();
|
||||
}
|
||||
|
||||
|
||||
Status drop_column(const std::string &column_name) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status create_all_vector_index(
|
||||
int concurrency, SegmentMeta::Ptr *new_segmnet_meta,
|
||||
std::unordered_map<std::string, VectorColumnIndexer::Ptr>
|
||||
*vector_indexers,
|
||||
std::unordered_map<std::string, VectorColumnIndexer::Ptr>
|
||||
*quant_vector_indexers) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status create_vector_index(
|
||||
const std::string &column, const IndexParams::Ptr &index_params,
|
||||
int concurrency, SegmentMeta::Ptr *new_segmnet_meta,
|
||||
std::unordered_map<std::string, VectorColumnIndexer::Ptr>
|
||||
*vector_indexers,
|
||||
std::unordered_map<std::string, VectorColumnIndexer::Ptr>
|
||||
*quant_vector_indexers) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status drop_vector_index(
|
||||
const std::string &column, SegmentMeta::Ptr *new_segmnet_meta,
|
||||
std::unordered_map<std::string, VectorColumnIndexer::Ptr>
|
||||
*vector_indexers) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status reload_vector_index(
|
||||
const CollectionSchema &schema, const SegmentMeta::Ptr &segment_meta,
|
||||
const std::unordered_map<std::string, VectorColumnIndexer::Ptr>
|
||||
&vector_indexers,
|
||||
const std::unordered_map<std::string, VectorColumnIndexer::Ptr>
|
||||
&quant_vector_indexers) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
bool vector_index_ready(const std::string &column,
|
||||
const IndexParams::Ptr &index_params) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool all_vector_index_ready() const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
Status create_scalar_index(
|
||||
const std::vector<std::string> &columns,
|
||||
const IndexParams::Ptr &index_params, SegmentMeta::Ptr *new_segment_meta,
|
||||
InvertedIndexer::Ptr *new_scalar_indexer) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status drop_scalar_index(const std::vector<std::string> &columns,
|
||||
SegmentMeta::Ptr *new_segment_meta,
|
||||
InvertedIndexer::Ptr *new_scalar_indexer) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status reload_scalar_index(
|
||||
const CollectionSchema &schema, const SegmentMeta::Ptr &segment_meta,
|
||||
const InvertedIndexer::Ptr &scalar_indexer) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Insert(Doc &doc) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Upsert(Doc &doc) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Update(Doc &doc) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Delete(const std::string &pk) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Delete(uint64_t doc_id) override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Doc::Ptr Fetch(uint64_t doc_id) override {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::vector<VectorColumnIndexer::Ptr> get_vector_indexer(
|
||||
const std::string &field_name) const override {
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<VectorColumnIndexer::Ptr> get_quant_vector_indexer(
|
||||
const std::string &field_name) const override {
|
||||
return {};
|
||||
}
|
||||
|
||||
Status flush() override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status dump() override {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status destroy() override {
|
||||
return Status::OK();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,322 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/sqlengine/analyzer/query_info_helper.h"
|
||||
#include "db/sqlengine/sqlengine_impl.h"
|
||||
#include "index/common/index_params.h"
|
||||
// #define private public
|
||||
#include <memory>
|
||||
#include "db/sqlengine/planner/optimizer.h"
|
||||
#include "mock_segment.h"
|
||||
// #undef private
|
||||
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
class MockInvertCondOptimizer : public InvertCondOptimizer {
|
||||
public:
|
||||
explicit MockInvertCondOptimizer(CollectionSchema *collection_schema)
|
||||
: InvertCondOptimizer(collection_schema) {}
|
||||
|
||||
public:
|
||||
bool invert_rule(Segment *segment, QueryRelNode *invert_cond) override;
|
||||
};
|
||||
|
||||
bool MockInvertCondOptimizer::invert_rule(Segment *segment,
|
||||
QueryRelNode *invert_cond) {
|
||||
if (invert_cond->op() == QueryNodeOp::Q_IN) {
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string invert_value = invert_cond->right()->text();
|
||||
|
||||
std::string numeric_text{""};
|
||||
QueryInfoHelper::data_buf_2_text(invert_value, DataType::INT32,
|
||||
&numeric_text);
|
||||
|
||||
int age = atoi(numeric_text.c_str());
|
||||
std::cout << "invert cond: age is " << age << std::endl;
|
||||
|
||||
// invert cond as less than 100
|
||||
if (age < 100) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
class OptimizerTest : public testing::Test {
|
||||
public:
|
||||
// Sets up the test fixture.
|
||||
static void SetUpTestSuite() {
|
||||
schema = std::make_shared<CollectionSchema>();
|
||||
auto &collection_schema_ = *schema;
|
||||
collection_schema_.set_name("collection");
|
||||
|
||||
// feature field
|
||||
auto column1 = std::make_shared<FieldSchema>();
|
||||
auto vector_params = std::make_shared<FlatIndexParams>(MetricType::IP);
|
||||
column1->set_name("face_feature");
|
||||
column1->set_index_params(vector_params);
|
||||
column1->set_dimension(4);
|
||||
column1->set_data_type(DataType::VECTOR_FP32);
|
||||
collection_schema_.add_field(column1);
|
||||
|
||||
// invert field
|
||||
auto column2 = std::make_shared<FieldSchema>();
|
||||
column2->set_name("age");
|
||||
column2->set_data_type(DataType::INT32);
|
||||
column2->set_index_params(std::make_shared<InvertIndexParams>(false));
|
||||
collection_schema_.add_field(column2);
|
||||
}
|
||||
|
||||
// Tears down the test fixture.
|
||||
static void TearDownTestSuite() {}
|
||||
|
||||
protected:
|
||||
inline static CollectionSchema::Ptr schema;
|
||||
Profiler::Ptr profiler_{new Profiler};
|
||||
};
|
||||
|
||||
|
||||
TEST_F(OptimizerTest, Basic) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = "age > 200";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr query_info = ret.value();
|
||||
|
||||
Optimizer::Ptr optimizer =
|
||||
std::make_shared<MockInvertCondOptimizer>(schema.get());
|
||||
|
||||
auto segment = std::make_shared<MockSegment>();
|
||||
|
||||
bool optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_TRUE(optimized);
|
||||
}
|
||||
|
||||
// case 1. invert subroot same as invert cond, do nothing
|
||||
TEST_F(OptimizerTest, Case1) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = "age > 12";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr query_info = ret.value();
|
||||
|
||||
Optimizer::Ptr optimizer =
|
||||
std::make_shared<MockInvertCondOptimizer>(schema.get());
|
||||
|
||||
auto segment = std::make_shared<MockSegment>();
|
||||
|
||||
bool optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_FALSE(optimized);
|
||||
}
|
||||
|
||||
// case 2.1 invert subroot is not found, all conds are forward cond
|
||||
TEST_F(OptimizerTest, Case2_1) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = "age > 100 and age > 101 or age > 102";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr query_info = ret.value();
|
||||
|
||||
Optimizer::Ptr optimizer =
|
||||
std::make_shared<MockInvertCondOptimizer>(schema.get());
|
||||
|
||||
auto segment = std::make_shared<MockSegment>();
|
||||
|
||||
bool optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_TRUE(optimized);
|
||||
}
|
||||
|
||||
// case 2.2 invert subroot is not found, some conds are forward cond
|
||||
// while left invert cond cannot be invert cond any more
|
||||
TEST_F(OptimizerTest, Case2_2) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = "age > 100 or age > 90";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr query_info = ret.value();
|
||||
|
||||
Optimizer::Ptr optimizer =
|
||||
std::make_shared<MockInvertCondOptimizer>(schema.get());
|
||||
|
||||
auto segment = std::make_shared<MockSegment>();
|
||||
|
||||
bool optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_FALSE(optimized);
|
||||
}
|
||||
|
||||
|
||||
// case 3.1 subroot is found and be part of invert cond
|
||||
TEST_F(OptimizerTest, Case3_1) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = "age > 100 and age > 101 and age > 10";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr query_info = ret.value();
|
||||
|
||||
Optimizer::Ptr optimizer =
|
||||
std::make_shared<MockInvertCondOptimizer>(schema.get());
|
||||
|
||||
auto segment = std::make_shared<MockSegment>();
|
||||
|
||||
bool optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_TRUE(optimized);
|
||||
ASSERT_TRUE(ret);
|
||||
}
|
||||
|
||||
// case 3.2 subroot is found and be part of invert cond
|
||||
TEST_F(OptimizerTest, Case3_2) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = "age > 10 and age > 11 and age > 100";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr query_info = ret.value();
|
||||
|
||||
Optimizer::Ptr optimizer =
|
||||
std::make_shared<MockInvertCondOptimizer>(schema.get());
|
||||
|
||||
auto segment = std::make_shared<MockSegment>();
|
||||
|
||||
bool optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_TRUE(optimized);
|
||||
}
|
||||
|
||||
// case 3.3 subroot is found and be part of invert cond
|
||||
TEST_F(OptimizerTest, Case3_3) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = "(age > 10 or age > 11) and age > 100";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr query_info = ret.value();
|
||||
|
||||
Optimizer::Ptr optimizer =
|
||||
std::make_shared<MockInvertCondOptimizer>(schema.get());
|
||||
|
||||
auto segment = std::make_shared<MockSegment>();
|
||||
|
||||
bool optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_TRUE(optimized);
|
||||
}
|
||||
|
||||
// case 3.4 subroot is found and be part of invert cond, but others also have
|
||||
// invert
|
||||
TEST_F(OptimizerTest, Case3_4) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = "age > 10 and (age > 101 and (age > 10 and age > 10))";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr query_info = ret.value();
|
||||
|
||||
Optimizer::Ptr optimizer =
|
||||
std::make_shared<MockInvertCondOptimizer>(schema.get());
|
||||
|
||||
auto segment = std::make_shared<MockSegment>();
|
||||
|
||||
bool optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_FALSE(optimized);
|
||||
}
|
||||
|
||||
|
||||
// case 4, optimize with in expr
|
||||
TEST_F(OptimizerTest, Case4) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = "age in (10, 20)";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr query_info = ret.value();
|
||||
|
||||
Optimizer::Ptr optimizer =
|
||||
std::make_shared<MockInvertCondOptimizer>(schema.get());
|
||||
|
||||
auto segment = std::make_shared<MockSegment>();
|
||||
|
||||
bool optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
// in will not optimized
|
||||
ASSERT_FALSE(optimized);
|
||||
|
||||
// in and optimizable, optimize optimizable
|
||||
query.filter_ = "age in (10, 20) and age > 100";
|
||||
ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
query_info = ret.value();
|
||||
optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_TRUE(optimized);
|
||||
|
||||
// in or optimizable, not optimized
|
||||
query.filter_ = "age in (10, 20) or age > 100";
|
||||
ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
query_info = ret.value();
|
||||
optimized = optimizer->optimize(segment.get(), query_info.get());
|
||||
ASSERT_FALSE(optimized);
|
||||
}
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
@@ -0,0 +1,679 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <gmock/gmock-matchers.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/sqlengine/sqlengine_impl.h"
|
||||
#include "index/common/doc.h"
|
||||
#include "index/common/schema.h"
|
||||
#include "profiler.h"
|
||||
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
class QueryInfoTest : public testing::Test {
|
||||
public:
|
||||
// Sets up the test fixture.
|
||||
static void SetUpTestSuite() {
|
||||
schema = std::make_shared<CollectionSchema>();
|
||||
auto ¶m = *schema;
|
||||
param.set_name("1collection");
|
||||
|
||||
auto column1 = std::make_shared<FieldSchema>();
|
||||
auto vector_params = std::make_shared<FlatIndexParams>(MetricType::IP);
|
||||
column1->set_name("face_feature");
|
||||
column1->set_index_params(vector_params);
|
||||
column1->set_dimension(4);
|
||||
column1->set_data_type(DataType::VECTOR_FP32);
|
||||
param.add_field(column1);
|
||||
|
||||
auto column2 = std::make_shared<FieldSchema>();
|
||||
column2->set_name("name");
|
||||
column2->set_data_type(DataType::UINT32);
|
||||
param.add_field(column2);
|
||||
|
||||
auto column3 = std::make_shared<FieldSchema>();
|
||||
column3->set_name("category");
|
||||
column3->set_data_type(DataType::STRING);
|
||||
param.add_field(column3);
|
||||
|
||||
auto column4 = std::make_shared<FieldSchema>();
|
||||
column4->set_name("face_feature");
|
||||
column4->set_dimension(4);
|
||||
column4->set_data_type(DataType::VECTOR_FP32);
|
||||
param.add_field(column4);
|
||||
|
||||
auto column5 = std::make_shared<FieldSchema>();
|
||||
column5->set_name("1-dash_score_field");
|
||||
column5->set_dimension(5);
|
||||
column5->set_data_type(DataType::STRING);
|
||||
param.add_field(column5);
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("name_array");
|
||||
column->set_data_type(DataType::ARRAY_UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("category_array");
|
||||
column->set_data_type(DataType::ARRAY_STRING);
|
||||
param.add_field(column);
|
||||
}
|
||||
}
|
||||
|
||||
// Tears down the test fixture.
|
||||
static void TearDownTestSuite() {}
|
||||
|
||||
protected:
|
||||
Profiler::Ptr profiler_{new Profiler};
|
||||
inline static CollectionSchema::Ptr schema;
|
||||
};
|
||||
|
||||
|
||||
TEST_F(QueryInfoTest, BasicQueryRequest) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value()) << ret.error().c_str();
|
||||
QueryInfo::Ptr new_query_info = ret.value();
|
||||
auto &query_fields = new_query_info->query_fields();
|
||||
EXPECT_EQ(query_fields.size(), 5);
|
||||
EXPECT_EQ(query_fields[0]->field_name(), "name");
|
||||
EXPECT_EQ(query_fields[1]->field_name(), "category");
|
||||
EXPECT_EQ(query_fields[2]->field_name(), "1-dash_score_field");
|
||||
EXPECT_EQ(query_fields[3]->field_name(), "name_array");
|
||||
EXPECT_EQ(query_fields[4]->field_name(), "category_array");
|
||||
EXPECT_EQ(new_query_info->query_topn(), 11);
|
||||
EXPECT_FALSE(new_query_info->filter_cond());
|
||||
EXPECT_FALSE(new_query_info->invert_cond());
|
||||
EXPECT_FALSE(new_query_info->post_filter_cond());
|
||||
EXPECT_FALSE(new_query_info->post_invert_cond());
|
||||
|
||||
ASSERT_TRUE(new_query_info->vector_cond_info());
|
||||
auto vector_cond = new_query_info->vector_cond_info();
|
||||
EXPECT_EQ(1, vector_cond->batch());
|
||||
EXPECT_EQ("face_feature", vector_cond->vector_field_name());
|
||||
EXPECT_EQ(query.query_vector_, vector_cond->vector_term());
|
||||
EXPECT_EQ(query.query_sparse_indices_, vector_cond->vector_sparse_indices());
|
||||
EXPECT_EQ(query.query_sparse_values_, vector_cond->vector_sparse_values());
|
||||
EXPECT_EQ(query.query_params_, vector_cond->query_params());
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, QueryRequestWithFilter) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
query.filter_ = "name<3 or name=4 or 1-dash_score_field='test'";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr new_query_info = ret.value();
|
||||
auto &query_fields = new_query_info->query_fields();
|
||||
EXPECT_EQ(query_fields.size(), 5);
|
||||
EXPECT_EQ(query_fields[0]->field_name(), "name");
|
||||
EXPECT_EQ(query_fields[1]->field_name(), "category");
|
||||
EXPECT_EQ(query_fields[2]->field_name(), "1-dash_score_field");
|
||||
EXPECT_EQ(query_fields[3]->field_name(), "name_array");
|
||||
EXPECT_EQ(query_fields[4]->field_name(), "category_array");
|
||||
EXPECT_EQ(new_query_info->query_topn(), 11);
|
||||
EXPECT_TRUE(new_query_info->filter_cond());
|
||||
EXPECT_FALSE(new_query_info->invert_cond());
|
||||
EXPECT_FALSE(new_query_info->post_filter_cond());
|
||||
EXPECT_FALSE(new_query_info->post_invert_cond());
|
||||
|
||||
ASSERT_TRUE(new_query_info->vector_cond_info());
|
||||
auto vector_cond = new_query_info->vector_cond_info();
|
||||
EXPECT_EQ(1, vector_cond->batch());
|
||||
EXPECT_EQ("face_feature", vector_cond->vector_field_name());
|
||||
EXPECT_EQ(query.query_vector_, vector_cond->vector_term());
|
||||
EXPECT_EQ(query.query_sparse_indices_, vector_cond->vector_sparse_indices());
|
||||
EXPECT_EQ(query.query_sparse_values_, vector_cond->vector_sparse_values());
|
||||
EXPECT_EQ(query.query_params_, vector_cond->query_params());
|
||||
|
||||
EXPECT_TRUE(new_query_info->filter_cond());
|
||||
// (nullptr) and (xxx)
|
||||
auto filter_cond = new_query_info->filter_cond();
|
||||
EXPECT_EQ(filter_cond->op_name(), "and");
|
||||
EXPECT_FALSE(filter_cond->left());
|
||||
|
||||
// ((name<3) or (name=4)) or (1-dash_score_field=test)
|
||||
auto right = std::dynamic_pointer_cast<QueryNode>(filter_cond->right());
|
||||
EXPECT_TRUE(right);
|
||||
EXPECT_EQ(right->op_name(), "or");
|
||||
|
||||
// 1-dash_score_field=test
|
||||
auto number_field_filter =
|
||||
std::dynamic_pointer_cast<QueryNode>(right->right());
|
||||
ASSERT_TRUE(number_field_filter);
|
||||
EXPECT_EQ(number_field_filter->op_name(), "=");
|
||||
auto left_key =
|
||||
std::dynamic_pointer_cast<QueryIDNode>(number_field_filter->left());
|
||||
EXPECT_EQ(left_key->op_name(), "ID");
|
||||
EXPECT_EQ(left_key->value(), "1-dash_score_field");
|
||||
auto right_const = std::dynamic_pointer_cast<QueryConstantNode>(
|
||||
number_field_filter->right());
|
||||
ASSERT_TRUE(right_const);
|
||||
EXPECT_EQ(right_const->op_name(), "STRING_VALUE");
|
||||
EXPECT_EQ(right_const->value(), "test");
|
||||
|
||||
// (name<3) or (name=4)
|
||||
auto left = std::dynamic_pointer_cast<QueryNode>(right->left());
|
||||
ASSERT_TRUE(left);
|
||||
EXPECT_EQ(left->op_name(), "or");
|
||||
auto or1 = std::dynamic_pointer_cast<QueryNode>(left->left());
|
||||
EXPECT_EQ(or1->op_name(), "<");
|
||||
auto id1 = std::dynamic_pointer_cast<QueryIDNode>(or1->left());
|
||||
ASSERT_TRUE(id1);
|
||||
EXPECT_EQ(id1->op_name(), "ID");
|
||||
EXPECT_EQ(id1->value(), "name");
|
||||
auto const1 = std::dynamic_pointer_cast<QueryConstantNode>(or1->right());
|
||||
ASSERT_TRUE(const1);
|
||||
EXPECT_EQ(const1->op_name(), "INT_VALUE");
|
||||
EXPECT_EQ(const1->value(), "3");
|
||||
auto or2 = std::dynamic_pointer_cast<QueryNode>(left->right());
|
||||
EXPECT_EQ(or2->op_name(), "=");
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, QueryRequestWithIncludeVector) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
query.include_vector_ = true;
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr new_query_info = ret.value();
|
||||
auto &query_fields = new_query_info->query_fields();
|
||||
EXPECT_EQ(query_fields.size(), 6);
|
||||
EXPECT_EQ(query_fields[0]->field_name(), "name");
|
||||
EXPECT_EQ(query_fields[1]->field_name(), "category");
|
||||
EXPECT_EQ(query_fields[2]->field_name(), "1-dash_score_field");
|
||||
EXPECT_EQ(query_fields[3]->field_name(), "name_array");
|
||||
EXPECT_EQ(query_fields[4]->field_name(), "category_array");
|
||||
EXPECT_EQ(query_fields[5]->field_name(), "face_feature");
|
||||
EXPECT_EQ(new_query_info->query_topn(), 11);
|
||||
EXPECT_FALSE(new_query_info->filter_cond());
|
||||
EXPECT_FALSE(new_query_info->invert_cond());
|
||||
EXPECT_FALSE(new_query_info->post_filter_cond());
|
||||
EXPECT_FALSE(new_query_info->post_invert_cond());
|
||||
|
||||
ASSERT_TRUE(new_query_info->vector_cond_info());
|
||||
auto vector_cond = new_query_info->vector_cond_info();
|
||||
EXPECT_EQ(1, vector_cond->batch());
|
||||
EXPECT_EQ("face_feature", vector_cond->vector_field_name());
|
||||
EXPECT_EQ(query.query_vector_, vector_cond->vector_term());
|
||||
EXPECT_EQ(query.query_sparse_indices_, vector_cond->vector_sparse_indices());
|
||||
EXPECT_EQ(query.query_sparse_values_, vector_cond->vector_sparse_values());
|
||||
EXPECT_EQ(query.query_params_, vector_cond->query_params());
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, OR_ANCESTOR) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
query.filter_ = "name=1 and (name=2 or name=3)";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr new_query_info = ret.value();
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, QueryRequestWithInFilter) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 10;
|
||||
query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
query.filter_ =
|
||||
"name=3 or name in (1, 2, 3) or category not in (\"a\", \"b\", \"c\")";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr new_query_info = ret.value();
|
||||
|
||||
auto &query_fields = new_query_info->query_fields();
|
||||
EXPECT_EQ(query_fields.size(), 5);
|
||||
EXPECT_EQ(query_fields[0]->field_name(), "name");
|
||||
EXPECT_EQ(query_fields[1]->field_name(), "category");
|
||||
EXPECT_EQ(query_fields[2]->field_name(), "1-dash_score_field");
|
||||
EXPECT_EQ(query_fields[3]->field_name(), "name_array");
|
||||
EXPECT_EQ(query_fields[4]->field_name(), "category_array");
|
||||
EXPECT_EQ(new_query_info->query_topn(), 10);
|
||||
|
||||
EXPECT_FALSE(new_query_info->invert_cond());
|
||||
EXPECT_FALSE(new_query_info->post_filter_cond());
|
||||
EXPECT_FALSE(new_query_info->post_invert_cond());
|
||||
|
||||
ASSERT_TRUE(new_query_info->vector_cond_info());
|
||||
auto vector_cond = new_query_info->vector_cond_info();
|
||||
EXPECT_EQ(1, vector_cond->batch());
|
||||
EXPECT_EQ("face_feature", vector_cond->vector_field_name());
|
||||
std::vector<float> data{1.1, 2.2, 3.3, 4.4};
|
||||
EXPECT_EQ(query.query_vector_, vector_cond->vector_term());
|
||||
|
||||
EXPECT_TRUE(new_query_info->filter_cond());
|
||||
// (nullptr) and (xxx)
|
||||
auto filter_cond = new_query_info->filter_cond();
|
||||
EXPECT_EQ(filter_cond->op_name(), "and");
|
||||
EXPECT_FALSE(filter_cond->left());
|
||||
|
||||
// ((name=3) or (name in (1, 2, 3))) or (category not in ("a", "b", "c"))
|
||||
auto right = std::dynamic_pointer_cast<QueryNode>(filter_cond->right());
|
||||
EXPECT_TRUE(right);
|
||||
EXPECT_EQ(right->op_name(), "or");
|
||||
|
||||
// category in ("a", "b", "c")
|
||||
auto category_filter = std::dynamic_pointer_cast<QueryNode>(right->right());
|
||||
ASSERT_TRUE(category_filter);
|
||||
EXPECT_EQ(category_filter->op_name(), " in ");
|
||||
auto left_key =
|
||||
std::dynamic_pointer_cast<QueryIDNode>(category_filter->left());
|
||||
EXPECT_EQ(left_key->op_name(), "ID");
|
||||
EXPECT_EQ(left_key->value(), "category");
|
||||
auto right_const =
|
||||
std::dynamic_pointer_cast<QueryListNode>(category_filter->right());
|
||||
ASSERT_TRUE(right_const);
|
||||
EXPECT_EQ(right_const->op_name(), "LIST_VALUE");
|
||||
EXPECT_EQ(right_const->text(), "NOT (a, b, c)");
|
||||
|
||||
// (name=3) or (name in (1, 2, 3))
|
||||
auto left = std::dynamic_pointer_cast<QueryNode>(right->left());
|
||||
ASSERT_TRUE(left);
|
||||
EXPECT_EQ(left->op_name(), "or");
|
||||
auto or1 = std::dynamic_pointer_cast<QueryNode>(left->left());
|
||||
EXPECT_EQ(or1->op_name(), "=");
|
||||
auto id1 = std::dynamic_pointer_cast<QueryIDNode>(or1->left());
|
||||
ASSERT_TRUE(id1);
|
||||
EXPECT_EQ(id1->op_name(), "ID");
|
||||
EXPECT_EQ(id1->value(), "name");
|
||||
auto const1 = std::dynamic_pointer_cast<QueryConstantNode>(or1->right());
|
||||
ASSERT_TRUE(const1);
|
||||
EXPECT_EQ(const1->op_name(), "INT_VALUE");
|
||||
EXPECT_EQ(const1->value(), "3");
|
||||
|
||||
auto or2 = std::dynamic_pointer_cast<QueryNode>(left->right());
|
||||
EXPECT_EQ(or2->op_name(), " in ");
|
||||
auto id2 = std::dynamic_pointer_cast<QueryIDNode>(or2->left());
|
||||
ASSERT_TRUE(id2);
|
||||
EXPECT_EQ(id2->op_name(), "ID");
|
||||
EXPECT_EQ(id2->value(), "name");
|
||||
auto const2 = std::dynamic_pointer_cast<QueryListNode>(or2->right());
|
||||
ASSERT_TRUE(const2);
|
||||
EXPECT_EQ(const2->op_name(), "LIST_VALUE");
|
||||
EXPECT_EQ(const2->text(), "(1, 2, 3)");
|
||||
}
|
||||
|
||||
|
||||
TEST_F(QueryInfoTest, QueryRequestWithInFilterWrong) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
|
||||
query.filter_ = ("name in ()");
|
||||
ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
|
||||
query.filter_ = ("name in (\"a\", 2, 3)");
|
||||
ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
|
||||
query.filter_ = ("name in (1.1, 2, 3)");
|
||||
ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
|
||||
query.filter_ = ("category in (1.1, \"b\")");
|
||||
ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, QueryRequestWithInFilterNum1024) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 10;
|
||||
query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
|
||||
std::string filter_str;
|
||||
for (int i = 0; i < 1024; i++) {
|
||||
if (i != 0) {
|
||||
filter_str += " or ";
|
||||
}
|
||||
filter_str += "name=" + std::to_string(i);
|
||||
}
|
||||
query.filter_ = filter_str;
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr new_query_info = ret.value();
|
||||
|
||||
auto &query_fields = new_query_info->query_fields();
|
||||
EXPECT_EQ(query_fields.size(), 5);
|
||||
EXPECT_EQ(query_fields[0]->field_name(), "name");
|
||||
EXPECT_EQ(query_fields[1]->field_name(), "category");
|
||||
EXPECT_EQ(query_fields[2]->field_name(), "1-dash_score_field");
|
||||
EXPECT_EQ(query_fields[3]->field_name(), "name_array");
|
||||
EXPECT_EQ(query_fields[4]->field_name(), "category_array");
|
||||
EXPECT_EQ(new_query_info->query_topn(), 10);
|
||||
|
||||
EXPECT_FALSE(new_query_info->invert_cond());
|
||||
EXPECT_FALSE(new_query_info->post_filter_cond());
|
||||
EXPECT_FALSE(new_query_info->post_invert_cond());
|
||||
|
||||
ASSERT_TRUE(new_query_info->vector_cond_info());
|
||||
auto vector_cond = new_query_info->vector_cond_info();
|
||||
EXPECT_EQ(1, vector_cond->batch());
|
||||
EXPECT_EQ("face_feature", vector_cond->vector_field_name());
|
||||
}
|
||||
|
||||
|
||||
TEST_F(QueryInfoTest, QueryRequestWithFilter_contain) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 10;
|
||||
query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "face_feature";
|
||||
query.include_vector_ = false;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
query.filter_ =
|
||||
R"( name_array contain_all (1, 2, 3) and )"
|
||||
R"( (name_array not contain_all (4, 5) or category_array contain_any
|
||||
("a", "b")) )"
|
||||
R"( or category_array not contain_any ("c", "d", "e")
|
||||
)";
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr new_query_info = ret.value();
|
||||
auto &query_fields = new_query_info->query_fields();
|
||||
// pre-defined schema field
|
||||
EXPECT_EQ(query_fields.size(), 5);
|
||||
EXPECT_EQ(query_fields[0]->field_name(), "name");
|
||||
EXPECT_EQ(query_fields[1]->field_name(), "category");
|
||||
EXPECT_EQ(query_fields[2]->field_name(), "1-dash_score_field");
|
||||
EXPECT_EQ(query_fields[3]->field_name(), "name_array");
|
||||
EXPECT_EQ(query_fields[4]->field_name(), "category_array");
|
||||
EXPECT_EQ(new_query_info->query_topn(), 10);
|
||||
|
||||
EXPECT_FALSE(new_query_info->invert_cond());
|
||||
EXPECT_FALSE(new_query_info->post_filter_cond());
|
||||
EXPECT_FALSE(new_query_info->post_invert_cond());
|
||||
|
||||
ASSERT_TRUE(new_query_info->vector_cond_info());
|
||||
auto vector_cond = new_query_info->vector_cond_info();
|
||||
EXPECT_EQ(1, vector_cond->batch());
|
||||
EXPECT_EQ("face_feature", vector_cond->vector_field_name());
|
||||
|
||||
EXPECT_TRUE(new_query_info->filter_cond());
|
||||
/*
|
||||
_________________[and]__________________
|
||||
/ \
|
||||
[nullptr(vector_cond)] [filter condition]
|
||||
*/
|
||||
// (nullptr) and (xxx)
|
||||
auto filter_cond = new_query_info->filter_cond();
|
||||
EXPECT_EQ(filter_cond->op_name(), "and");
|
||||
EXPECT_FALSE(filter_cond->left());
|
||||
|
||||
/*
|
||||
_______________[or]_______________
|
||||
/ \
|
||||
_____________[and]_____________ [category_array not
|
||||
contain_any
|
||||
("c", "d", "e")]
|
||||
/ \
|
||||
[name_array contain_all (1, 2, 3)] ___________[or]______________
|
||||
/ \
|
||||
[name_array not contain_all (4, 5)] [category_array
|
||||
contain_any ("a", "b")]
|
||||
*/
|
||||
// name_array contain_all (1, 2, 3) and
|
||||
// (name_array not contain_all (4, 5) or category_array contain_any ("a",
|
||||
// "b")) or category_array not contain_any ("c", "d", "e")
|
||||
auto parent_node = std::dynamic_pointer_cast<QueryNode>(filter_cond);
|
||||
auto cur_node = std::dynamic_pointer_cast<QueryNode>(filter_cond->right());
|
||||
EXPECT_TRUE(cur_node);
|
||||
EXPECT_EQ(cur_node->op_name(), "or");
|
||||
|
||||
|
||||
// category_array not contain_any ("c", "d", "e")
|
||||
parent_node = std::dynamic_pointer_cast<QueryNode>(cur_node);
|
||||
cur_node = std::dynamic_pointer_cast<QueryNode>(cur_node->right());
|
||||
EXPECT_TRUE(cur_node);
|
||||
EXPECT_EQ(cur_node->op_name(), " contain_any ");
|
||||
{
|
||||
auto left_key = std::dynamic_pointer_cast<QueryIDNode>(cur_node->left());
|
||||
EXPECT_EQ(left_key->op_name(), "ID");
|
||||
EXPECT_EQ(left_key->value(), "category_array");
|
||||
auto right_const =
|
||||
std::dynamic_pointer_cast<QueryListNode>(cur_node->right());
|
||||
ASSERT_TRUE(right_const);
|
||||
EXPECT_EQ(right_const->op_name(), "LIST_VALUE");
|
||||
EXPECT_EQ(right_const->text(), "NOT (c, d, e)");
|
||||
}
|
||||
cur_node = parent_node;
|
||||
|
||||
// name_array contain_all (1, 2, 3) and
|
||||
// (name_array not contain_all (4, 5) or category_array contain_any ("a",
|
||||
// "b"))
|
||||
parent_node = std::dynamic_pointer_cast<QueryNode>(cur_node);
|
||||
cur_node = std::dynamic_pointer_cast<QueryNode>(cur_node->left());
|
||||
EXPECT_TRUE(cur_node);
|
||||
EXPECT_EQ(cur_node->op_name(), "and");
|
||||
|
||||
// the left side of 'and'
|
||||
// name_array contain_all (1, 2, 3)
|
||||
parent_node = std::dynamic_pointer_cast<QueryNode>(cur_node);
|
||||
cur_node = std::dynamic_pointer_cast<QueryNode>(cur_node->left());
|
||||
EXPECT_TRUE(cur_node);
|
||||
EXPECT_EQ(cur_node->op_name(), " contain_all ");
|
||||
{
|
||||
auto left_key = std::dynamic_pointer_cast<QueryIDNode>(cur_node->left());
|
||||
EXPECT_EQ(left_key->op_name(), "ID");
|
||||
EXPECT_EQ(left_key->value(), "name_array");
|
||||
auto right_const =
|
||||
std::dynamic_pointer_cast<QueryListNode>(cur_node->right());
|
||||
ASSERT_TRUE(right_const);
|
||||
EXPECT_EQ(right_const->op_name(), "LIST_VALUE");
|
||||
EXPECT_EQ(right_const->text(), "(1, 2, 3)");
|
||||
}
|
||||
cur_node = parent_node;
|
||||
|
||||
// the right side of 'and'
|
||||
// (name_array not contain_all (4, 5) or category_array contain_any ("a",
|
||||
// "b"))
|
||||
parent_node = std::dynamic_pointer_cast<QueryNode>(cur_node);
|
||||
cur_node = std::dynamic_pointer_cast<QueryNode>(cur_node->right());
|
||||
EXPECT_TRUE(cur_node);
|
||||
EXPECT_EQ(cur_node->op_name(), "or");
|
||||
|
||||
// name_array not contain_all (4, 5)
|
||||
parent_node = std::dynamic_pointer_cast<QueryNode>(cur_node);
|
||||
cur_node = std::dynamic_pointer_cast<QueryNode>(cur_node->left());
|
||||
EXPECT_TRUE(cur_node);
|
||||
EXPECT_EQ(cur_node->op_name(), " contain_all ");
|
||||
{
|
||||
auto left_key = std::dynamic_pointer_cast<QueryIDNode>(cur_node->left());
|
||||
EXPECT_EQ(left_key->op_name(), "ID");
|
||||
EXPECT_EQ(left_key->value(), "name_array");
|
||||
auto right_const =
|
||||
std::dynamic_pointer_cast<QueryListNode>(cur_node->right());
|
||||
ASSERT_TRUE(right_const);
|
||||
EXPECT_EQ(right_const->op_name(), "LIST_VALUE");
|
||||
EXPECT_EQ(right_const->text(), "NOT (4, 5)");
|
||||
}
|
||||
cur_node = parent_node;
|
||||
|
||||
// category_array contain_any ("a", "b"))
|
||||
parent_node = std::dynamic_pointer_cast<QueryNode>(cur_node);
|
||||
cur_node = std::dynamic_pointer_cast<QueryNode>(cur_node->right());
|
||||
EXPECT_TRUE(cur_node);
|
||||
EXPECT_EQ(cur_node->op_name(), " contain_any ");
|
||||
{
|
||||
auto left_key = std::dynamic_pointer_cast<QueryIDNode>(cur_node->left());
|
||||
EXPECT_EQ(left_key->op_name(), "ID");
|
||||
EXPECT_EQ(left_key->value(), "category_array");
|
||||
auto right_const =
|
||||
std::dynamic_pointer_cast<QueryListNode>(cur_node->right());
|
||||
ASSERT_TRUE(right_const);
|
||||
EXPECT_EQ(right_const->op_name(), "LIST_VALUE");
|
||||
EXPECT_EQ(right_const->text(), "(a, b)");
|
||||
}
|
||||
cur_node = parent_node;
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, SelectNonExistField) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"category_array", "not_exist_field"};
|
||||
query.topk_ = 11;
|
||||
query.include_vector_ = false;
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
EXPECT_THAT(ret.error().message(),
|
||||
testing::HasSubstr("not defined in schema"));
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, ContainAllExceedLimit) {
|
||||
VectorQuery query;
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name_array not contain_all (";
|
||||
for (int i = 0; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
EXPECT_THAT(ret.error().message(),
|
||||
testing::HasSubstr(
|
||||
"Contain_* rel expr only support list size no more than 32"));
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, ContainAnyExceedLimit) {
|
||||
VectorQuery query;
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "name_array not contain_any (";
|
||||
for (int i = 0; i <= 32; i++) {
|
||||
query.filter_ += std::to_string(i);
|
||||
if (i < 32) {
|
||||
query.filter_ += ", ";
|
||||
}
|
||||
}
|
||||
query.filter_ += ")";
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
EXPECT_THAT(ret.error().message(),
|
||||
testing::HasSubstr(
|
||||
"Contain_* rel expr only support list size no more than 32"));
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, ArrayLengthNonExistField) {
|
||||
VectorQuery query;
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "array_length(not_exist_field) > 1";
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
EXPECT_THAT(ret.error().message(),
|
||||
testing::HasSubstr("array_length argument not found in schema"));
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, ArrayLengthOnNonArrayField) {
|
||||
VectorQuery query;
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "array_length(name) > 1";
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
EXPECT_THAT(ret.error().message(),
|
||||
testing::HasSubstr("array_length only support array"));
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, ArrayLengthInvalidArgument) {
|
||||
VectorQuery query;
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "array_length(name_array) > '1'";
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
EXPECT_THAT(
|
||||
ret.error().message(),
|
||||
testing::HasSubstr("array_length right side only support integer"));
|
||||
}
|
||||
|
||||
TEST_F(QueryInfoTest, ArrayLengthInvalidOp) {
|
||||
VectorQuery query;
|
||||
query.topk_ = 200;
|
||||
query.filter_ = "array_length(name_array) like '%'";
|
||||
auto engine = std::make_shared<SQLEngineImpl>(std::make_shared<Profiler>());
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
ASSERT_FALSE(ret.has_value());
|
||||
EXPECT_THAT(ret.error().message(), testing::HasSubstr("syntax error"));
|
||||
}
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
@@ -0,0 +1,269 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <arrow/api.h>
|
||||
#include <arrow/io/api.h>
|
||||
#include <arrow/ipc/api.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/common/file_helper.h"
|
||||
#include "db/index/common/version_manager.h"
|
||||
#include "db/index/segment/segment.h"
|
||||
#include "index/common/index_params.h"
|
||||
#include "index/common/schema.h"
|
||||
#include "index/common/type.h"
|
||||
|
||||
namespace zvec {
|
||||
|
||||
inline CollectionSchema::Ptr GetCollectionSchema() {
|
||||
auto invert_params = std::make_shared<InvertIndexParams>(true);
|
||||
auto collection_schema = std::make_shared<CollectionSchema>(
|
||||
"test_collection",
|
||||
std::vector<FieldSchema::Ptr>{
|
||||
std::make_shared<FieldSchema>("id", DataType::UINT64, false, nullptr),
|
||||
std::make_shared<FieldSchema>("invert_id", DataType::UINT64, false,
|
||||
invert_params),
|
||||
|
||||
std::make_shared<FieldSchema>("bool", DataType::BOOL, false, nullptr),
|
||||
std::make_shared<FieldSchema>("invert_bool", DataType::BOOL, false,
|
||||
invert_params),
|
||||
|
||||
std::make_shared<FieldSchema>("bool_array", DataType::ARRAY_BOOL,
|
||||
false, nullptr),
|
||||
std::make_shared<FieldSchema>(
|
||||
"invert_bool_array", DataType::ARRAY_BOOL, false, invert_params),
|
||||
|
||||
std::make_shared<FieldSchema>("name", DataType::STRING, false,
|
||||
nullptr),
|
||||
std::make_shared<FieldSchema>("invert_name", DataType::STRING, false,
|
||||
invert_params),
|
||||
|
||||
std::make_shared<FieldSchema>("age", DataType::INT32, false, nullptr),
|
||||
std::make_shared<FieldSchema>(
|
||||
"invert_age", DataType::INT32, false,
|
||||
std::make_shared<InvertIndexParams>(true)),
|
||||
|
||||
std::make_shared<FieldSchema>("score", DataType::DOUBLE, false,
|
||||
nullptr),
|
||||
|
||||
std::make_shared<FieldSchema>("optional_age", DataType::UINT32, true,
|
||||
nullptr),
|
||||
std::make_shared<FieldSchema>("invert_optional_age", DataType::UINT32,
|
||||
true, invert_params),
|
||||
|
||||
std::make_shared<FieldSchema>("category_set", DataType::ARRAY_INT32,
|
||||
true, nullptr),
|
||||
std::make_shared<FieldSchema>("invert_category_set",
|
||||
DataType::ARRAY_INT32, true,
|
||||
invert_params),
|
||||
|
||||
// add vector field
|
||||
std::make_shared<FieldSchema>(
|
||||
"dense", DataType::VECTOR_FP32, 4, false,
|
||||
std::make_shared<FlatIndexParams>(MetricType::L2)),
|
||||
|
||||
// add sparse vector
|
||||
std::make_shared<FieldSchema>(
|
||||
"sparse", DataType::SPARSE_VECTOR_FP32, 0, false,
|
||||
std::make_shared<FlatIndexParams>(MetricType::IP)),
|
||||
});
|
||||
|
||||
return collection_schema;
|
||||
}
|
||||
|
||||
inline Doc CreateDoc(const uint64_t doc_id) {
|
||||
Doc new_doc;
|
||||
new_doc.set_pk("pk_" + std::to_string(doc_id));
|
||||
new_doc.set_doc_id(doc_id);
|
||||
|
||||
new_doc.set<uint64_t>("id", doc_id);
|
||||
new_doc.set<uint64_t>("invert_id", doc_id);
|
||||
new_doc.set<bool>("bool", doc_id % 100 == 0);
|
||||
new_doc.set<bool>("invert_bool", doc_id % 100 == 0);
|
||||
new_doc.set<int32_t>("age", doc_id % 100);
|
||||
new_doc.set<int32_t>("invert_age", doc_id % 100);
|
||||
if (uint32_t v = doc_id % 100; v) {
|
||||
new_doc.set("optional_age", v);
|
||||
new_doc.set("invert_optional_age", v);
|
||||
}
|
||||
auto name = "user_" + std::to_string(doc_id % 100);
|
||||
new_doc.set<std::string>("name", name);
|
||||
new_doc.set<std::string>("invert_name", name);
|
||||
new_doc.set<double>("score", static_cast<double>(rand() % 1000) / 10.0);
|
||||
|
||||
// vector
|
||||
std::vector<float> vv;
|
||||
for (uint32_t i = 0; i < 4; i++) {
|
||||
vv.push_back(static_cast<float>(doc_id));
|
||||
}
|
||||
new_doc.set<std::vector<float>>("dense", vv);
|
||||
|
||||
// sparse vector
|
||||
{
|
||||
std::vector<uint32_t> indices;
|
||||
std::vector<float> values;
|
||||
for (uint32_t i = 0; i < doc_id % 100; i++) {
|
||||
indices.push_back(i);
|
||||
values.push_back(static_cast<float>(doc_id));
|
||||
}
|
||||
new_doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
||||
"sparse", std::make_pair(indices, values));
|
||||
}
|
||||
|
||||
auto category_size = doc_id % 100;
|
||||
if (category_size > 0) {
|
||||
std::vector<int32_t> category;
|
||||
for (uint32_t i = 1; i <= category_size; i++) {
|
||||
category.push_back(i);
|
||||
}
|
||||
new_doc.set("category_set", category);
|
||||
new_doc.set("invert_category_set", category);
|
||||
}
|
||||
|
||||
if (doc_id % 3 == 0) {
|
||||
new_doc.set<std::vector<bool>>("bool_array", {true, false, true});
|
||||
new_doc.set<std::vector<bool>>("invert_bool_array", {true, false, true});
|
||||
} else if (doc_id % 3 == 1) {
|
||||
new_doc.set<std::vector<bool>>("bool_array", {true, true, true});
|
||||
new_doc.set<std::vector<bool>>("invert_bool_array", {true, true, true});
|
||||
} else {
|
||||
new_doc.set<std::vector<bool>>("bool_array", {false, false, false});
|
||||
new_doc.set<std::vector<bool>>("invert_bool_array", {false, false, false});
|
||||
}
|
||||
|
||||
return new_doc;
|
||||
}
|
||||
|
||||
inline Status InsertDoc(const Segment::Ptr &segment,
|
||||
const uint64_t start_doc_id,
|
||||
const uint64_t end_doc_id) {
|
||||
srand(time(NULL));
|
||||
long long create_total = 0;
|
||||
long long insert_total = 0;
|
||||
for (auto doc_id = start_doc_id; doc_id < end_doc_id; doc_id++) {
|
||||
if (segment) {
|
||||
auto start = std::chrono::system_clock::now();
|
||||
Doc new_doc = CreateDoc(doc_id);
|
||||
auto end = std::chrono::system_clock::now();
|
||||
auto create_cost =
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
|
||||
.count();
|
||||
create_total += create_cost;
|
||||
|
||||
start = std::chrono::system_clock::now();
|
||||
auto status = segment->Insert(new_doc);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
end = std::chrono::system_clock::now();
|
||||
auto insert_cost =
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
|
||||
.count();
|
||||
insert_total += insert_cost;
|
||||
}
|
||||
}
|
||||
std::cout << "pure create cost " << create_total << "us" << std::endl;
|
||||
std::cout << "pure insert cost " << insert_total << "us" << std::endl;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
class RecallTest : public testing::Test {
|
||||
protected:
|
||||
static void SetUpTestSuite() {
|
||||
FileHelper::RemoveDirectory(seg_path_);
|
||||
FileHelper::CreateDirectory(seg_path_);
|
||||
|
||||
collection_schema_ = GetCollectionSchema();
|
||||
auto segment = create_segment();
|
||||
if (segment == nullptr) {
|
||||
LOG_ERROR("create segment failed");
|
||||
EXPECT_TRUE(segment != nullptr);
|
||||
std::exit(EXIT_FAILURE);
|
||||
}
|
||||
auto status = InsertDoc(segment, 0, 10000);
|
||||
if (!status.ok()) {
|
||||
LOG_ERROR("insert doc failed: %s", status.c_str());
|
||||
EXPECT_TRUE(status.ok());
|
||||
std::exit(EXIT_FAILURE);
|
||||
}
|
||||
segments_.push_back(segment);
|
||||
}
|
||||
|
||||
static void TearDownTestSuite() {
|
||||
segments_.clear();
|
||||
FileHelper::RemoveDirectory(seg_path_);
|
||||
}
|
||||
|
||||
public:
|
||||
static std::string GetPath() {
|
||||
return seg_path_;
|
||||
}
|
||||
|
||||
static Segment::Ptr create_segment();
|
||||
|
||||
protected:
|
||||
static inline std::string seg_path_ = "./test_collection";
|
||||
static inline CollectionSchema::Ptr collection_schema_;
|
||||
static inline std::vector<Segment::Ptr> segments_;
|
||||
};
|
||||
|
||||
inline Segment::Ptr RecallTest::create_segment() {
|
||||
auto seg_path = GetPath();
|
||||
auto segment_meta = std::make_shared<SegmentMeta>();
|
||||
segment_meta->set_id(0);
|
||||
|
||||
auto id_map = IDMap::CreateAndOpen("test_collection", GetPath() + "/id_map",
|
||||
true, false);
|
||||
auto delete_store = std::make_shared<DeleteStore>("test_collection");
|
||||
|
||||
Version v1;
|
||||
v1.set_schema(*collection_schema_);
|
||||
std::string v_path = GetPath() + "/test_manifest";
|
||||
FileHelper::CreateDirectory(v_path);
|
||||
auto vm = VersionManager::Create(v_path, v1);
|
||||
if (!vm.has_value()) {
|
||||
LOG_ERROR("create version manager failed: %s", vm.error().c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
BlockMeta mem_block;
|
||||
mem_block.id_ = 0;
|
||||
mem_block.type_ = BlockType::SCALAR;
|
||||
mem_block.min_doc_id_ = 0;
|
||||
mem_block.max_doc_id_ = 0;
|
||||
mem_block.doc_count_ = 0;
|
||||
segment_meta->set_writing_forward_block(mem_block);
|
||||
|
||||
SegmentOptions options;
|
||||
options.read_only_ = false;
|
||||
options.enable_mmap_ = true;
|
||||
|
||||
auto result =
|
||||
Segment::CreateAndOpen(GetPath(), *collection_schema_, 0, 0, id_map,
|
||||
delete_store, vm.value(), options);
|
||||
|
||||
if (!result) {
|
||||
LOG_ERROR("create segment failed: %s", result.error().c_str());
|
||||
return nullptr;
|
||||
}
|
||||
auto segment = result.value();
|
||||
return segment;
|
||||
}
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,650 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "sqlengine/analyzer/simple_rewriter.h"
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/index/common/doc.h"
|
||||
#include "db/index/common/schema.h"
|
||||
#include "db/sqlengine/analyzer/query_info.h"
|
||||
#include "db/sqlengine/sqlengine_impl.h"
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
class SimpleRewriterTest : public testing::Test {
|
||||
public:
|
||||
// Sets up the test fixture.
|
||||
static void SetUpTestSuite() {
|
||||
schema = std::make_shared<CollectionSchema>();
|
||||
auto ¶m = *schema;
|
||||
param.set_name("1collection");
|
||||
|
||||
auto column1 = std::make_shared<FieldSchema>();
|
||||
auto vector_params = std::make_shared<FlatIndexParams>(MetricType::IP);
|
||||
column1->set_name("face_feature");
|
||||
column1->set_index_params(vector_params);
|
||||
column1->set_dimension(4);
|
||||
column1->set_data_type(DataType::VECTOR_FP32);
|
||||
param.add_field(column1);
|
||||
|
||||
auto column2 = std::make_shared<FieldSchema>();
|
||||
column2->set_name("age");
|
||||
column2->set_data_type(DataType::UINT32);
|
||||
param.add_field(column2);
|
||||
|
||||
auto column_gender = std::make_shared<FieldSchema>();
|
||||
column_gender->set_name("gender");
|
||||
column_gender->set_data_type(DataType::UINT32);
|
||||
param.add_field(column_gender);
|
||||
|
||||
auto column3 = std::make_shared<FieldSchema>();
|
||||
column3->set_name("category");
|
||||
column3->set_data_type(DataType::STRING);
|
||||
param.add_field(column3);
|
||||
|
||||
auto column4 = std::make_shared<FieldSchema>();
|
||||
column4->set_name("face_feature");
|
||||
column4->set_dimension(4);
|
||||
column4->set_data_type(DataType::VECTOR_FP32);
|
||||
param.add_field(column4);
|
||||
|
||||
auto column5 = std::make_shared<FieldSchema>();
|
||||
column5->set_name("filename");
|
||||
column5->set_dimension(5);
|
||||
column5->set_data_type(DataType::STRING);
|
||||
param.add_field(column5);
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("loc");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("fid");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("agent_id");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("state");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("categoryId");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("passed_days");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("category_in");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("category_out");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("intAttr");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("intAttr");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("strAttr");
|
||||
column->set_data_type(DataType::STRING);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("partitionName");
|
||||
column->set_data_type(DataType::STRING);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("doc_id");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("a");
|
||||
column->set_data_type(DataType::UINT32);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("is_type1");
|
||||
column->set_data_type(DataType::BOOL);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("is_type2");
|
||||
column->set_data_type(DataType::BOOL);
|
||||
param.add_field(column);
|
||||
}
|
||||
|
||||
{
|
||||
auto column = std::make_shared<FieldSchema>();
|
||||
column->set_name("category_array");
|
||||
column->set_data_type(DataType::ARRAY_STRING);
|
||||
param.add_field(column);
|
||||
}
|
||||
}
|
||||
|
||||
// Tears down the test fixture.
|
||||
static void TearDownTestSuite() {}
|
||||
|
||||
QueryInfo::Ptr parse(const std::string &filter) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"*"};
|
||||
query.topk_ = 11;
|
||||
query.include_vector_ = false;
|
||||
query.filter_ = filter;
|
||||
|
||||
auto engine = std::make_shared<SQLEngineImpl>(profiler_);
|
||||
auto ret = engine->parse_request(schema, query, nullptr);
|
||||
|
||||
// ASSERT_TRUE(ret.has_value());
|
||||
QueryInfo::Ptr new_query_info = ret.value();
|
||||
return new_query_info;
|
||||
}
|
||||
|
||||
|
||||
protected:
|
||||
Profiler::Ptr profiler_{new Profiler};
|
||||
inline static CollectionSchema::Ptr schema;
|
||||
};
|
||||
|
||||
class EqOrRewriteTest : public SimpleRewriterTest {};
|
||||
|
||||
TEST_F(EqOrRewriteTest, SimpleEqOr) {
|
||||
auto info = parse("age = 10 or age = 20 ");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(), "age in (10, 20)(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, SimpleManyEqOr) {
|
||||
auto info = parse(
|
||||
"age = 1 or age = 2 or age = 3 or age = 4 "
|
||||
"or age = 5 or age = 6 or age = 7 or age = 8 or age = 9 or age = 10 or "
|
||||
"age = 11 or age = 12 or age = 13 or age = 14 or age = 15 or age = 16 or "
|
||||
"age = 17 or age = 18 or age = 19 or age = 20 or age = 21 or age = 22 or "
|
||||
"age = 23 or age = 24 or age = 25 or age = 26 or age = 27 or age = 28 or "
|
||||
"age = 29 or age = 30 or age = 31 or age = 32 or age = 33 or age = 34 or "
|
||||
"age = 35 or age = 36 or age = 37 or age = 38 or age = 39 or age = 40 or "
|
||||
"age = 41 or age = 42 or age = 43 or age = 44 or age = 45 or age = 46 or "
|
||||
"age = 47 or age = 48 or age = 49 or age = 50 or age = 51 or age = 52 or "
|
||||
"age = 53 or age = 54 or age = 55 or age = 56 or age = 57 or age = 58 or "
|
||||
"age = 59 or age = 60 or age = 61 or age = 62 or age = 63 or age = 64 or "
|
||||
"age = 65 or age = 66 or age = 67 or age = 68 or age = 69 or age = 70 or "
|
||||
"age = 71 or age = 72 or age = 73 or age = 74 or age = 75 or age = 76 or "
|
||||
"age = 77 or age = 78 or age = 79 or age = 80 or age = 81 or age = 82 or "
|
||||
"age = 83 or age = 84 or age = 85 or age = 86 or age = 87 or age = 88 or "
|
||||
"age = 89 or age = 90 or age = 91 or age = 92 or age = 93 or age = 94 or "
|
||||
"age = 95 or age = 96 or age = 97 or age = 98 or age = 99 or age = 100");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(
|
||||
info->filter_cond()->text(),
|
||||
"age in (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, "
|
||||
"19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, "
|
||||
"37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, "
|
||||
"55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, "
|
||||
"73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, "
|
||||
"91, 92, 93, 94, 95, 96, 97, 98, 99, 100)(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, SimpleManyEqOrParas) {
|
||||
auto info = parse(
|
||||
"age = 1 or age = 2 or age = 3 or age = 4 "
|
||||
"or age = 5 or age = 6 or (age = 7 or age = 8 or age = 9 or age = 10 or "
|
||||
"age = 11 or age = 12 or age = 13) or age = 14 or age = 15 or age = 16 "
|
||||
"or "
|
||||
"age = 17 or age = 18 or age = 19 or age = 20 or age = 21 or age = 22 or "
|
||||
"age = 23 or age = 24 or age = 25 or age = 26 or age = 27 or age = 28 or "
|
||||
"age = 29 or age = 30 or age = 31 or age = 32 or age = 33 or age = 34 or "
|
||||
"age = 35 or age = 36 or age = 37 or (age = 38 or age = 39 or age = 40 "
|
||||
"or "
|
||||
"age = 41 or age = 42 or age = 43 or age = 44 or age = 45 or age = 46 or "
|
||||
"age = 47 or age = 48 or age = 49 or age = 50 or age = 51 or age = 52 or "
|
||||
"age = 53 or age = 54 or age = 55 or age = 56 or age = 57 or age = 58 or "
|
||||
"age = 59 or age = 60 or age = 61 or age = 62 or age = 63 or age = 64 or "
|
||||
"age = 65 or age = 66 or age = 67 or age = 68 or age = 69 or age = 70 or "
|
||||
"age = 71 or age = 72 or age = 73 or age = 74 or age = 75 or age = 76 or "
|
||||
"age = 77 or age = 78 or age = 79 or age = 80 or age = 81 or age = 82 or "
|
||||
"age = 83 or age = 84 or age = 85) or age = 86 or age = 87 or age = 88 "
|
||||
"or "
|
||||
"age = 89 or age = 90 or age = 91 or age = 92 or age = 93 or age = 94 or "
|
||||
"age = 95 or age = 96 or age = 97 or (age = 98 or age = 99) or age = "
|
||||
"100");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(
|
||||
info->filter_cond()->text(),
|
||||
"age in (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, "
|
||||
"19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, "
|
||||
"37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, "
|
||||
"55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, "
|
||||
"73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, "
|
||||
"91, 92, 93, 94, 95, 96, 97, 98, 99, 100)(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, SimpleNeOr) {
|
||||
auto info = parse("age != 10 or age != 20 ");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(), "age in NOT (10, 20)(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, SimpleManyNeOr) {
|
||||
auto info = parse(
|
||||
"age != 1 or age != 2 or age != 3 or age "
|
||||
"!= 4 or age != 5 or age != 6 or age != 7 or age != 8 or age != 9 or age "
|
||||
"!= 10 or age != 11 or age != 12 or age != 13 or age != 14 or age != 15 "
|
||||
"or age != 16 or age != 17 or age != 18 or age != 19 or age != 20");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"age in NOT (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, "
|
||||
"16, 17, 18, "
|
||||
"19, 20)(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, EqAndNe) {
|
||||
auto info = parse(
|
||||
"age != 10 or age != 20 or age = 30 or "
|
||||
"age = 40");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(age in NOT (10, 20)(FORWARD)(OR_A)) or (age in (30, "
|
||||
"40)(FORWARD)(OR_A))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, PreEqOr) {
|
||||
{
|
||||
auto info = parse(
|
||||
"gender =1 or age = 10 or age = 20 or "
|
||||
"age = 30 or age = 40");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(gender=1(FORWARD)(OR_A)) or (age in (10, 20, 30, "
|
||||
"40)(FORWARD)(OR_A))");
|
||||
}
|
||||
{
|
||||
auto info = parse(
|
||||
"gender =1 and age = 10 or age = 20 or "
|
||||
"age = 30 or age = 40");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"((gender=1(FORWARD)(OR_A)) and (age=10(FORWARD)(OR_A))) or (age "
|
||||
"in (20, 30, 40)(FORWARD)(OR_A))");
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, PostEqOr) {
|
||||
{
|
||||
auto info = parse(
|
||||
"age = 10 or age = 20 or "
|
||||
"age = 30 or age = 40 or gender = 1");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(age in (10, 20, 30, 40)(FORWARD)(OR_A)) or "
|
||||
"(gender=1(FORWARD)(OR_A))");
|
||||
}
|
||||
{
|
||||
auto info = parse(
|
||||
"age = 10 or age = 20 or "
|
||||
"age = 30 or age = 40 and gender = 1");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(age in (10, 20, 30)(FORWARD)(OR_A)) or "
|
||||
"((age=40(FORWARD)(OR_A)) and (gender=1(FORWARD)(OR_A)))");
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, PreEqAnd) {
|
||||
auto info = parse(
|
||||
"gender =1 and (age = 10 or age = 20 or "
|
||||
"age = 30 or age = 40)");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(gender=1(FORWARD)) and (age in (10, 20, 30, 40)(FORWARD))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, PostEqAnd) {
|
||||
auto info = parse(
|
||||
"(age = 10 or age = 20 or "
|
||||
"age = 30 or age = 40) and gender=1");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(age in (10, 20, 30, 40)(FORWARD)) and (gender=1(FORWARD))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, PrePostEqAnd) {
|
||||
auto info = parse(
|
||||
"gender =1 and (age = 10 or age = 20 or "
|
||||
"age = 30 or age = 40) and loc != 3");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"((gender=1(FORWARD)) and (age in (10, 20, 30, 40)(FORWARD))) and "
|
||||
"(loc!=3(FORWARD))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, UserCases1) {
|
||||
auto info = parse(
|
||||
"(agent_id=20) and state=1 and (fid=107 "
|
||||
"or fid=174 or fid=593 or fid=602 or fid=592 or fid=134 or fid=135 or "
|
||||
"fid=136 or fid=137 or fid=138 or fid=139 or fid=141 or fid=267 or "
|
||||
"fid=271 or fid=176 or fid=177 or fid=178 or fid=179 or fid=180 or "
|
||||
"fid=182 or fid=183 or fid=184 or fid=270 or fid=479 or fid=488 or "
|
||||
"fid=502 or fid=508 or fid=522 or fid=553 or fid=554 or fid=557 or "
|
||||
"fid=561 or fid=567 or fid=570 or fid=588 or fid=594 or fid=595 or "
|
||||
"fid=596 or fid=597 or fid=598 or fid=603 or fid=604 or fid=605 or "
|
||||
"fid=606 or fid=426 or fid=427 or fid=428 or fid=429 or fid=430 or "
|
||||
"fid=431 or fid=432 or fid=433 or fid=434 or fid=435 or fid=436 or "
|
||||
"fid=437 or fid=438 or fid=439 or fid=440 or fid=441 or fid=442 or "
|
||||
"fid=443 or fid=444 or fid=445 or fid=446 or fid=447 or fid=448 or "
|
||||
"fid=215 or fid=216 or fid=217 or fid=469 or fid=473 or fid=475 or "
|
||||
"fid=476 or fid=477 or fid=478 or fid=524 or fid=528 or fid=529 or "
|
||||
"fid=532 or fid=533 or fid=534 or fid=542 or fid=543 or fid=560 or "
|
||||
"fid=243 or fid=244 or fid=245 or fid=246 or fid=247 or fid=496 or "
|
||||
"fid=497 or fid=506 or fid=248 or fid=249 or fid=250 or fid=251 or "
|
||||
"fid=252 or fid=494 or fid=495 or fid=507 or fid=535 or fid=536 or "
|
||||
"fid=586 or fid=589 or fid=259 or fid=260 or fid=261 or fid=262 or "
|
||||
"fid=263 or fid=264 or fid=265 or fid=491 or fid=492 or fid=493 or "
|
||||
"fid=530 or fid=531 or fid=227 or fid=228 or fid=229 or fid=230 or "
|
||||
"fid=231 or fid=232 or fid=233 or fid=235 or fid=472 or fid=487 or "
|
||||
"fid=537 or fid=559 or fid=236 or fid=237 or fid=238 or fid=239 or "
|
||||
"fid=240 or fid=241 or fid=242 or fid=273 or fid=546 or fid=587 or "
|
||||
"fid=454 or fid=455 or fid=456 or fid=457 or fid=458 or fid=459 or "
|
||||
"fid=460 or fid=461 or fid=449 or fid=450 or fid=451 or fid=452 or "
|
||||
"fid=453 or fid=480 or fid=481 or fid=482 or fid=483 or fid=484 or "
|
||||
"fid=489 or fid=490 or fid=538 or fid=539 or fid=540 or fid=545 or "
|
||||
"fid=503 or fid=504 or fid=547 or fid=548 or fid=549 or fid=550 or "
|
||||
"fid=509 or fid=510 or fid=511 or fid=512 or fid=513 or fid=523 or "
|
||||
"fid=558 or fid=555 or fid=556 or fid=600 or fid=601 or fid=562 or "
|
||||
"fid=563 or fid=564 or fid=565 or fid=566 or fid=591 or fid=568 or "
|
||||
"fid=569 or fid=590 or fid=571 or fid=572 or fid=573 or fid=574 or "
|
||||
"fid=575 or fid=701 or fid=711 or fid=713 or fid=616 or fid=617 or "
|
||||
"fid=618 or fid=619 or fid=620 or fid=621 or fid=622 or fid=623 or "
|
||||
"fid=624 or fid=625 or fid=626 or fid=629 or fid=672 or fid=607 or "
|
||||
"fid=700 or fid=635 or fid=612 or fid=613 or fid=614 or fid=615 or "
|
||||
"fid=679 or fid=670 or fid=680 or fid=681 or fid=702 or fid=706 or "
|
||||
"fid=714 or fid=675 or fid=676 or fid=640 or fid=643 or fid=649 or "
|
||||
"fid=653 or fid=655 or fid=657 or fid=662 or fid=703 or fid=704 or "
|
||||
"fid=705 or fid=707 or fid=641 or fid=642 or fid=644 or fid=645 or "
|
||||
"fid=646 or fid=647 or fid=648 or fid=709 or fid=650 or fid=651 or "
|
||||
"fid=652 or fid=710 or fid=654 or fid=656 or fid=658 or fid=659 or "
|
||||
"fid=660 or fid=661 or fid=663 or fid=664 or fid=665 or fid=666 or "
|
||||
"fid=667 or fid=668 or fid=669 or fid=678)");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(
|
||||
info->filter_cond()->text(),
|
||||
"((agent_id=20(FORWARD)) and (state=1(FORWARD))) and (fid in (107, 174, "
|
||||
"593, 602, 592, 134, 135, 136, 137, 138, 139, 141, 267, 271, 176, 177, "
|
||||
"178, 179, 180, 182, 183, 184, 270, 479, 488, 502, 508, 522, 553, 554, "
|
||||
"557, 561, 567, 570, 588, 594, 595, 596, 597, 598, 603, 604, 605, 606, "
|
||||
"426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, "
|
||||
"440, 441, 442, 443, 444, 445, 446, 447, 448, 215, 216, 217, 469, 473, "
|
||||
"475, 476, 477, 478, 524, 528, 529, 532, 533, 534, 542, 543, 560, 243, "
|
||||
"244, 245, 246, 247, 496, 497, 506, 248, 249, 250, 251, 252, 494, 495, "
|
||||
"507, 535, 536, 586, 589, 259, 260, 261, 262, 263, 264, 265, 491, 492, "
|
||||
"493, 530, 531, 227, 228, 229, 230, 231, 232, 233, 235, 472, 487, 537, "
|
||||
"559, 236, 237, 238, 239, 240, 241, 242, 273, 546, 587, 454, 455, 456, "
|
||||
"457, 458, 459, 460, 461, 449, 450, 451, 452, 453, 480, 481, 482, 483, "
|
||||
"484, 489, 490, 538, 539, 540, 545, 503, 504, 547, 548, 549, 550, 509, "
|
||||
"510, 511, 512, 513, 523, 558, 555, 556, 600, 601, 562, 563, 564, 565, "
|
||||
"566, 591, 568, 569, 590, 571, 572, 573, 574, 575, 701, 711, 713, 616, "
|
||||
"617, 618, 619, 620, 621, 622, 623, 624, 625, 626, 629, 672, 607, 700, "
|
||||
"635, 612, 613, 614, 615, 679, 670, 680, 681, 702, 706, 714, 675, 676, "
|
||||
"640, 643, 649, 653, 655, 657, 662, 703, 704, 705, 707, 641, 642, 644, "
|
||||
"645, 646, 647, 648, 709, 650, 651, 652, 710, 654, 656, 658, 659, 660, "
|
||||
"661, 663, 664, 665, 666, 667, 668, 669, 678)(FORWARD))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, UserCases2) {
|
||||
auto info = parse(
|
||||
"partitionName = '114634' or "
|
||||
"partitionName = '114632' or partitionName = '114635' or partitionName = "
|
||||
"'114629' or partitionName = '114630' or partitionName = '114633' or "
|
||||
"partitionName = '114636' or partitionName = '114637' or partitionName = "
|
||||
"'114631'");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"partitionName in (114634, 114632, 114635, 114629, 114630, 114633, "
|
||||
"114636, 114637, 114631)(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, UserCases3) {
|
||||
auto info = parse(
|
||||
"(doc_id=1319620650600837120 or "
|
||||
"doc_id=1319621497753739264 or doc_id=1319629144649367552 or "
|
||||
"doc_id=1319630319721377793 or doc_id=1319667286769324032 or "
|
||||
"doc_id=1319671157117808640 or doc_id=1319671403998793728 or "
|
||||
"doc_id=2319684930499055617 or doc_id=1319685259995140096)");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"doc_id in (1319620650600837120, 1319621497753739264, "
|
||||
"1319629144649367552, 1319630319721377793, 1319667286769324032, "
|
||||
"1319671157117808640, 1319671403998793728, 2319684930499055617, "
|
||||
"1319685259995140096)(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, UserCases4) {
|
||||
auto info = parse(
|
||||
"(strAttr ='' or strAttr = 'prd') and "
|
||||
"categoryId = 4");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(strAttr in (, prd)(FORWARD)) and (categoryId=4(FORWARD))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, UserCases5) {
|
||||
auto info = parse(
|
||||
"intAttr = 1 OR intAttr = 5 OR intAttr "
|
||||
"= 6 OR intAttr = 9 and categoryId = 1");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(intAttr in (1, 5, 6)(FORWARD)(OR_A)) or "
|
||||
"((intAttr=9(FORWARD)(OR_A)) and (categoryId=1(FORWARD)(OR_A)))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, UserCases6) {
|
||||
auto info = parse(
|
||||
""
|
||||
"filename='OhbVrpoi.pdf' or "
|
||||
"filename='wRyoG4dB.pdf' or "
|
||||
"filename='dJ3fawFf.pdf' or "
|
||||
"filename='ZJS9dk3Q.pdf' or "
|
||||
"filename='fY2JD8dL.pdf' or "
|
||||
"filename='HnJpdoxC.pdf' or "
|
||||
"filename='Hbxm1zvi.pdf' or "
|
||||
"filename='r5Q8cxHu.pdf' or "
|
||||
"filename='dwF9cZtI.pdf'");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"filename in (OhbVrpoi.pdf, "
|
||||
"wRyoG4dB.pdf, "
|
||||
"dJ3fawFf.pdf, "
|
||||
"ZJS9dk3Q.pdf, "
|
||||
"fY2JD8dL.pdf, "
|
||||
"HnJpdoxC.pdf, "
|
||||
"Hbxm1zvi.pdf, "
|
||||
"r5Q8cxHu.pdf, "
|
||||
"dwF9cZtI.pdf)(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, NotChanged1) {
|
||||
auto info = parse(
|
||||
"passed_days>3 and (loc >= "
|
||||
"500 or age > 10)");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(passed_days>3(FORWARD)) and ((loc>=500(FORWARD)(OR_A)) "
|
||||
"or (age>10(FORWARD)(OR_A)))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, NotChanged2) {
|
||||
auto info = parse(
|
||||
"strAttr=\"online_252\" AND (intAttr > "
|
||||
"103775813 OR intAttr < 103775813) and categoryId = 88888888");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(
|
||||
info->filter_cond()->text(),
|
||||
"((strAttr=online_252(FORWARD)) and ((intAttr>103775813(FORWARD)(OR_A)) "
|
||||
"or (intAttr<103775813(FORWARD)(OR_A)))) and "
|
||||
"(categoryId=88888888(FORWARD))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, NotChanged3) {
|
||||
auto info = parse(
|
||||
"(is_type1 = true or is_type2 = "
|
||||
"true)");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(is_type1=true(FORWARD)(OR_A)) or (is_type2=true(FORWARD)(OR_A))");
|
||||
}
|
||||
|
||||
TEST_F(EqOrRewriteTest, NotChanged4) {
|
||||
auto info = parse("(a = 1 or a != 2)");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(a=1(FORWARD)(OR_A)) or (a!=2(FORWARD)(OR_A))");
|
||||
}
|
||||
|
||||
class ContainRewriteTest : public SimpleRewriterTest {};
|
||||
|
||||
TEST_F(ContainRewriteTest, ContainAllEmptySet) {
|
||||
auto info = parse("category_array contain_all ()");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"category_array IS_NOT_NULL (FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(ContainRewriteTest, NotContainAllEmptySet) {
|
||||
auto info = parse("category_array not contain_all ()");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->is_filter_unsatisfiable(), true);
|
||||
}
|
||||
|
||||
TEST_F(ContainRewriteTest, NotContainAnyEmptySet) {
|
||||
auto info = parse("category_array not contain_any ()");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"category_array IS_NOT_NULL (FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(ContainRewriteTest, ContainAnyEmptySet) {
|
||||
auto info = parse("category_array contain_any ()");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->is_filter_unsatisfiable(), true);
|
||||
}
|
||||
|
||||
TEST_F(ContainRewriteTest, AlwaysFalseConditionAnd) {
|
||||
auto info = parse("category_array not contain_all () and a = 1");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->is_filter_unsatisfiable(), true);
|
||||
}
|
||||
|
||||
TEST_F(ContainRewriteTest, AlwaysFalseConditionMultiAnd) {
|
||||
auto info = parse(
|
||||
"category_array not contain_all () and a > 1 and a > 2 and a > 3 and a > "
|
||||
"4");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->is_filter_unsatisfiable(), true);
|
||||
}
|
||||
|
||||
TEST_F(ContainRewriteTest, AlwaysFalseConditionOr) {
|
||||
auto info = parse("category_array not contain_all () or a = 1");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->filter_cond()->text(), "a=1(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(ContainRewriteTest, AlwaysFalseConditionMultiOr) {
|
||||
auto info =
|
||||
parse("category_array not contain_all () or a > 1 or a > 2 or a > 3");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(
|
||||
info->filter_cond()->text(),
|
||||
"((a>1(FORWARD)(OR_A)) or (a>2(FORWARD)(OR_A))) or (a>3(FORWARD)(OR_A))");
|
||||
}
|
||||
|
||||
TEST_F(ContainRewriteTest, AlwaysFalseConditionAndComplex) {
|
||||
auto info = parse("(a > 1 or a < 0) and category_array contain_any () ");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->is_filter_unsatisfiable(), true);
|
||||
}
|
||||
|
||||
TEST_F(ContainRewriteTest, AlwaysFalseConditionOrComplex) {
|
||||
auto info = parse("(a > 1 or a < 0) or category_array contain_any () ");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->is_filter_unsatisfiable(), false);
|
||||
EXPECT_EQ(info->filter_cond()->text(),
|
||||
"(a>1(FORWARD)(OR_A)) or (a<0(FORWARD)(OR_A))");
|
||||
}
|
||||
|
||||
TEST_F(SimpleRewriterTest, MiscOr) {
|
||||
auto info = parse("a = 1 or a = 2 or a = 3 or category_array contain_any ()");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->is_filter_unsatisfiable(), false);
|
||||
EXPECT_EQ(info->filter_cond()->text(), "a in (1, 2, 3)(FORWARD)");
|
||||
}
|
||||
|
||||
TEST_F(SimpleRewriterTest, MiscAnd) {
|
||||
auto info =
|
||||
parse("(a = 1 or a = 2 or a = 3) and category_array contain_any ()");
|
||||
ASSERT_NE(info, nullptr);
|
||||
EXPECT_EQ(info->is_filter_unsatisfiable(), true);
|
||||
}
|
||||
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
@@ -0,0 +1,170 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "db/sqlengine/sqlengine.h"
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <gtest/gtest.h>
|
||||
#include "index/common/query_params.h"
|
||||
#include "index/common/schema.h"
|
||||
#include "index/common/type.h"
|
||||
#include "mock_segment.h"
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
class SqlEngineTest : public testing::Test {
|
||||
public:
|
||||
void SetUp() override {
|
||||
auto vector_params = std::make_shared<FlatIndexParams>(MetricType::IP);
|
||||
schema_ = std::make_shared<CollectionSchema>(
|
||||
"test_collection",
|
||||
std::vector<FieldSchema::Ptr>{
|
||||
std::make_shared<FieldSchema>("id", DataType::INT32, false, 0,
|
||||
nullptr),
|
||||
std::make_shared<FieldSchema>(
|
||||
"name", DataType::STRING, false, 0, // nullptr
|
||||
std::make_shared<InvertIndexParams>(false)),
|
||||
std::make_shared<FieldSchema>("age", DataType::INT64, false, 0,
|
||||
nullptr),
|
||||
std::make_shared<FieldSchema>("score", DataType::DOUBLE, false, 0,
|
||||
nullptr),
|
||||
std::make_shared<FieldSchema>("tag_list", DataType::ARRAY_INT32,
|
||||
false, 0, nullptr),
|
||||
std::make_shared<FieldSchema>("vector",
|
||||
DataType::SPARSE_VECTOR_FP32, false,
|
||||
4, vector_params),
|
||||
});
|
||||
}
|
||||
|
||||
protected:
|
||||
CollectionSchema::Ptr schema_;
|
||||
};
|
||||
|
||||
TEST_F(SqlEngineTest, Forward) {
|
||||
std::vector<Segment::Ptr> segments = {std::make_shared<MockSegment>()};
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age", "tag_list"};
|
||||
query.topk_ = 11;
|
||||
// query.filter_ = "id > 3 and score < 0.1";
|
||||
// query.filter_ = "name like 'name_2%'";
|
||||
// query.filter_ = "name not in ('name_2','name_4')";
|
||||
// query.filter_ = "tag_list contain_all (1,2,3,4)";
|
||||
query.filter_ = "tag_list is null";
|
||||
if (const char *env_var = std::getenv("FILTER"); env_var != nullptr) {
|
||||
query.filter_ = env_var;
|
||||
}
|
||||
query.include_vector_ = true;
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(schema_, query, segments);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
EXPECT_TRUE(ret.has_value());
|
||||
}
|
||||
|
||||
TEST_F(SqlEngineTest, Vector) {
|
||||
std::vector<Segment::Ptr> segments = {std::make_shared<MockSegment>()};
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "score"};
|
||||
query.topk_ = 11;
|
||||
query.filter_ = "id > 3 and score < 0.1";
|
||||
if (const char *env_var = std::getenv("FILTER"); env_var != nullptr) {
|
||||
query.filter_ = env_var;
|
||||
}
|
||||
// query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.query_sparse_indices_ = "[0, 1, 2, 3]";
|
||||
query.query_sparse_values_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "vector";
|
||||
query.include_vector_ = true;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(schema_, query, segments);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
EXPECT_TRUE(ret.has_value());
|
||||
}
|
||||
|
||||
TEST_F(SqlEngineTest, Invert) {
|
||||
std::vector<Segment::Ptr> segments = {std::make_shared<MockSegment>()};
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "age", "score"};
|
||||
query.topk_ = 11;
|
||||
// query.filter_ = "name = 'test_name'";
|
||||
query.filter_ = "name is not null";
|
||||
if (const char *env_var = std::getenv("FILTER"); env_var != nullptr) {
|
||||
query.filter_ = env_var;
|
||||
}
|
||||
query.include_vector_ = true;
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(schema_, query, segments);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
EXPECT_TRUE(ret.has_value());
|
||||
}
|
||||
|
||||
TEST_F(SqlEngineTest, MultiSegments) {
|
||||
std::vector<Segment::Ptr> segments = {std::make_shared<MockSegment>(),
|
||||
std::make_shared<MockSegment>()};
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age", "score"};
|
||||
query.topk_ = 11;
|
||||
query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "vector";
|
||||
// query.filter_ = "name = 'test_name'";
|
||||
if (const char *env_var = std::getenv("FILTER"); env_var != nullptr) {
|
||||
query.filter_ = env_var;
|
||||
}
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(schema_, query, segments);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
EXPECT_TRUE(ret.has_value());
|
||||
}
|
||||
|
||||
TEST_F(SqlEngineTest, GroupBy) {
|
||||
std::vector<Segment::Ptr> segments = {std::make_shared<MockSegment>()};
|
||||
GroupByVectorQuery query;
|
||||
query.group_by_field_name_ = "name";
|
||||
query.group_count_ = 3;
|
||||
query.group_topk_ = 2;
|
||||
query.output_fields_ = {"id", "name", "score"};
|
||||
query.filter_ = "id > 3 and score < 0.1";
|
||||
if (const char *env_var = std::getenv("FILTER"); env_var != nullptr) {
|
||||
query.filter_ = env_var;
|
||||
}
|
||||
// query.query_vector_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.query_sparse_indices_ = "[0, 1, 2, 3]";
|
||||
query.query_sparse_values_ = "[0.1, 0.2, 0.3, 0.4]";
|
||||
query.field_name_ = "vector";
|
||||
query.include_vector_ = true;
|
||||
query.query_params_ = std::make_shared<QueryParams>(IndexType::FLAT);
|
||||
query.query_params_->set_radius(0.8F);
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute_group_by(schema_, query, segments);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
EXPECT_TRUE(ret.has_value());
|
||||
}
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
@@ -0,0 +1,112 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <arrow/api.h>
|
||||
#include <arrow/io/api.h>
|
||||
#include <arrow/ipc/api.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/common/file_helper.h"
|
||||
#include "db/index/common/version_manager.h"
|
||||
#include "db/index/segment/segment.h"
|
||||
#include "db/sqlengine/sqlengine.h"
|
||||
#include "index/common/index_params.h"
|
||||
#include "index/common/schema.h"
|
||||
#include "index/common/type.h"
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
using CreateDocFun = Doc (*)(const uint64_t doc_id);
|
||||
|
||||
inline Status InsertDoc(const Segment::Ptr &segment,
|
||||
const uint64_t start_doc_id, const uint64_t end_doc_id,
|
||||
CreateDocFun create_doc) {
|
||||
srand(time(NULL));
|
||||
long long create_total = 0;
|
||||
long long insert_total = 0;
|
||||
for (auto doc_id = start_doc_id; doc_id < end_doc_id; doc_id++) {
|
||||
if (segment) {
|
||||
auto start = std::chrono::system_clock::now();
|
||||
Doc new_doc = create_doc(doc_id);
|
||||
auto end = std::chrono::system_clock::now();
|
||||
auto create_cost =
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
|
||||
.count();
|
||||
create_total += create_cost;
|
||||
|
||||
start = std::chrono::system_clock::now();
|
||||
auto status = segment->Insert(new_doc);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
end = std::chrono::system_clock::now();
|
||||
auto insert_cost =
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
|
||||
.count();
|
||||
insert_total += insert_cost;
|
||||
}
|
||||
}
|
||||
std::cout << "pure create cost " << create_total << "us" << std::endl;
|
||||
std::cout << "pure insert cost " << insert_total << "us" << std::endl;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
inline Segment::Ptr create_segment(const std::string &seg_path,
|
||||
const CollectionSchema &schema) {
|
||||
auto segment_meta = std::make_shared<SegmentMeta>();
|
||||
segment_meta->set_id(0);
|
||||
|
||||
auto id_map = IDMap::CreateAndOpen("test_collection", seg_path + "/id_map",
|
||||
true, false);
|
||||
auto delete_store = std::make_shared<DeleteStore>("test_collection");
|
||||
|
||||
Version v1;
|
||||
v1.set_schema(schema);
|
||||
std::string v_path = seg_path + "/test_manifest";
|
||||
FileHelper::CreateDirectory(v_path);
|
||||
auto vm = VersionManager::Create(v_path, v1);
|
||||
if (!vm.has_value()) {
|
||||
LOG_ERROR("create version manager failed: %s", vm.error().c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
BlockMeta mem_block;
|
||||
mem_block.id_ = 0;
|
||||
mem_block.type_ = BlockType::SCALAR;
|
||||
mem_block.min_doc_id_ = 0;
|
||||
mem_block.max_doc_id_ = 0;
|
||||
mem_block.doc_count_ = 0;
|
||||
segment_meta->set_writing_forward_block(mem_block);
|
||||
|
||||
SegmentOptions options;
|
||||
options.read_only_ = false;
|
||||
options.enable_mmap_ = true;
|
||||
|
||||
auto result = Segment::CreateAndOpen(seg_path, schema, 0, 0, id_map,
|
||||
delete_store, vm.value(), options);
|
||||
|
||||
if (!result) {
|
||||
LOG_ERROR("create segment failed: %s", result.error().c_str());
|
||||
return nullptr;
|
||||
}
|
||||
auto segment = result.value();
|
||||
return segment;
|
||||
}
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
@@ -0,0 +1,216 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <gtest/gtest.h>
|
||||
#include "db/sqlengine/sqlengine.h"
|
||||
#include "index/common/schema.h"
|
||||
#include "recall_base.h"
|
||||
|
||||
namespace zvec::sqlengine {
|
||||
|
||||
class VectorRecallTest : public RecallTest {};
|
||||
|
||||
TEST_F(VectorRecallTest, Basic) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
std::vector<float> feature(4, 0.0);
|
||||
query.query_vector_.assign((const char *)feature.data(),
|
||||
feature.size() * sizeof(float));
|
||||
query.field_name_ = "dense";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int i = 0; i < query.topk_; i++) {
|
||||
auto &doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
EXPECT_FLOAT_EQ(doc->score(), (float)i * i * 4);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(VectorRecallTest, HybridInvertFilter) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.filter_ = "invert_id >= 1";
|
||||
query.topk_ = 200;
|
||||
std::vector<float> feature(4, 0.0);
|
||||
query.query_vector_.assign((const char *)feature.data(),
|
||||
feature.size() * sizeof(float));
|
||||
query.field_name_ = "dense";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int j = 0; j < query.topk_; j++) {
|
||||
auto &doc = docs[j];
|
||||
int i = j + 1;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
EXPECT_FLOAT_EQ(doc->score(), (float)i * i * 4);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(VectorRecallTest, HybridInvertFilterBfByKeys) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.filter_ = "invert_id < 199";
|
||||
query.topk_ = 199;
|
||||
std::vector<float> feature(4, 0.0);
|
||||
query.query_vector_.assign((const char *)feature.data(),
|
||||
feature.size() * sizeof(float));
|
||||
query.field_name_ = "dense";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int i = 0; i < query.topk_; i++) {
|
||||
auto &doc = docs[i];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
EXPECT_FLOAT_EQ(doc->score(), (float)i * i * 4);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(VectorRecallTest, HybridForwardFilter) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.filter_ = "id >= 1";
|
||||
query.topk_ = 200;
|
||||
std::vector<float> feature(4, 0.0);
|
||||
query.query_vector_.assign((const char *)feature.data(),
|
||||
feature.size() * sizeof(float));
|
||||
query.field_name_ = "dense";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
for (int j = 0; j < query.topk_; j++) {
|
||||
auto &doc = docs[j];
|
||||
int i = j + 1;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(i));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), i % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(i % 100));
|
||||
EXPECT_FLOAT_EQ(doc->score(), (float)i * i * 4);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(VectorRecallTest, HybridInvertForwardFilter) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"name", "age"};
|
||||
query.filter_ = "invert_id >= 1 and id <= 100";
|
||||
query.topk_ = 200;
|
||||
std::vector<float> feature(4, 0.0);
|
||||
query.query_vector_.assign((const char *)feature.data(),
|
||||
feature.size() * sizeof(float));
|
||||
query.field_name_ = "dense";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), 100);
|
||||
for (size_t j = 0; j < docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
int doc_id = j + 1;
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), doc_id % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(doc_id % 100));
|
||||
EXPECT_FLOAT_EQ(doc->score(), (float)doc_id * doc_id * 4);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(VectorRecallTest, Sparse) {
|
||||
VectorQuery query;
|
||||
query.output_fields_ = {"id", "name", "age"};
|
||||
query.topk_ = 200;
|
||||
std::vector<float> feature(4, 1.0);
|
||||
std::vector<uint32_t> indices{0, 1, 2, 3};
|
||||
query.query_sparse_indices_.assign((const char *)indices.data(),
|
||||
indices.size() * sizeof(uint32_t));
|
||||
query.query_sparse_values_.assign((const char *)feature.data(),
|
||||
feature.size() * sizeof(float));
|
||||
query.field_name_ = "sparse";
|
||||
|
||||
auto engine = SQLEngine::create(std::make_shared<Profiler>());
|
||||
auto ret = engine->execute(collection_schema_, query, segments_);
|
||||
if (!ret) {
|
||||
LOG_ERROR("execute failed: [%s]", ret.error().c_str());
|
||||
}
|
||||
ASSERT_TRUE(ret.has_value());
|
||||
auto docs = ret.value();
|
||||
EXPECT_EQ(docs.size(), query.topk_);
|
||||
|
||||
int doc_id = 9999;
|
||||
for (size_t j = 0; j < docs.size(); j++) {
|
||||
auto &doc = docs[j];
|
||||
EXPECT_EQ(doc->pk(), "pk_" + std::to_string(doc_id));
|
||||
auto age = doc->get<int32_t>("age");
|
||||
EXPECT_EQ(age.value(), doc_id % 100);
|
||||
auto name = doc->get<std::string>("name");
|
||||
ASSERT_TRUE(name);
|
||||
EXPECT_EQ(name.value(), "user_" + std::to_string(doc_id % 100));
|
||||
EXPECT_FLOAT_EQ(doc->score(), (float)doc_id * 4);
|
||||
doc_id--;
|
||||
while (doc_id % 100 <= 3) {
|
||||
doc_id--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace zvec::sqlengine
|
||||
Reference in New Issue
Block a user