diff --git a/src/binding/c/c_api.cc b/src/binding/c/c_api.cc index 9144836..5645386 100644 --- a/src/binding/c/c_api.cc +++ b/src/binding/c/c_api.cc @@ -4307,7 +4307,7 @@ size_t zvec_doc_memory_usage(const zvec_doc_t *doc) { return doc_ptr->memory_usage();) } -zvec_error_code_t zvec_doc_validate(const zvec_doc_t *doc, +zvec_error_code_t zvec_doc_validate_and_sanitize(zvec_doc_t *doc, const zvec_collection_schema_t *schema, bool is_update, char **error_msg) { if (!doc || !schema) { @@ -4327,15 +4327,15 @@ zvec_error_code_t zvec_doc_validate(const zvec_doc_t *doc, return status_to_error_code(status); } - auto doc_ptr = reinterpret_cast(doc); - status = doc_ptr->validate(schema_ptr, is_update); if (!status.ok()) { + auto doc_ptr = reinterpret_cast(doc); + status = doc_ptr->validate_and_sanitize(schema_ptr, is_update); if (!status.ok()) { if (error_msg) { *error_msg = copy_string(status.message()); } return status_to_error_code(status); } - if (error_msg) { *error_msg = nullptr; } + if (error_msg) { *error_msg = nullptr; } return ZVEC_OK;) } diff --git a/src/db/collection.cc b/src/db/collection.cc index 7cae6fc..8673a92 100644 --- a/src/db/collection.cc +++ b/src/db/collection.cc @@ -35,7 +35,6 @@ #include "db/common/file_helper.h" #include "db/common/profiler.h" #include "db/common/typedef.h" -#include "db/index/column/vector_column/vector_column_indexer.h" #include "db/index/common/delete_store.h" #include "db/index/common/id_map.h" #include "db/index/common/index_filter.h" @@ -1443,8 +1442,8 @@ Result CollectionImpl::write_impl(std::vector &docs, CHECK_DESTROY_RETURN_STATUS_EXPECTED(destroyed_, false); for (auto &&doc : docs) { - auto validate = doc.validate(schema_, mode == WriteMode::UPDATE); - CHECK_RETURN_STATUS_EXPECTED(validate); + auto s = doc.validate_and_sanitize(schema_, mode == WriteMode::UPDATE); + CHECK_RETURN_STATUS_EXPECTED(s); } // TODO: The granularity of the write_lock is too coarse. @@ -1458,7 +1457,6 @@ Result CollectionImpl::write_impl(std::vector &docs, kMaxWriteBatchSize)); } - // validate docs for (auto &&doc : docs) { if (need_switch_to_new_segment()) { auto s = switch_to_new_segment_for_writing(); @@ -1583,7 +1581,9 @@ Result CollectionImpl::Query(const VectorQuery &query) const { CHECK_DESTROY_RETURN_STATUS_EXPECTED(destroyed_, false); - auto s = query.validate(schema_->get_vector_field(query.field_name_)); + VectorQuery sanitized = query; + auto s = sanitized.validate_and_sanitize( + schema_->get_vector_field(sanitized.field_name_)); CHECK_RETURN_STATUS_EXPECTED(s); auto segments = get_all_segments(); @@ -1591,7 +1591,7 @@ Result CollectionImpl::Query(const VectorQuery &query) const { return DocPtrList(); } - return sql_engine_->execute(schema_, query, segments); + return sql_engine_->execute(schema_, sanitized, segments); } Result CollectionImpl::GroupByQuery( diff --git a/src/db/index/common/doc.cc b/src/db/index/common/doc.cc index bcb401d..0405eac 100644 --- a/src/db/index/common/doc.cc +++ b/src/db/index/common/doc.cc @@ -11,10 +11,13 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + +#include #include #include #include #include +#include #include #include #include @@ -114,6 +117,9 @@ std::string get_value_type_name(const Doc::Value &value, bool is_vector) { value); } + +namespace { + template T byte_swap(T value) { if constexpr (std::is_same_v) { @@ -159,6 +165,68 @@ T read_value_from_buffer(const uint8_t *&data) { return value; } +template +std::string vec_to_string(const std::vector &v) { + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < v.size(); ++i) { + if (i > 0) oss << ", "; + oss << +v[i]; // + from print as char + } + oss << "]"; + return oss.str(); +} + +template +struct overloaded : Ts... { + using Ts::operator()...; +}; + +template +overloaded(Ts...) -> overloaded; + + +bool sort_and_find_duplicates(uint32_t *indices, char *values, size_t n, + size_t value_byte_size) { + if (n <= 1) { + return false; + } + bool already_sorted = true; + for (size_t i = 1; i < n; ++i) { + if (indices[i] == indices[i - 1]) { + return true; + } + if (indices[i] < indices[i - 1]) { + already_sorted = false; + break; + } + } + if (already_sorted) { + return false; + } + std::vector perm(n); + std::iota(perm.begin(), perm.end(), size_t{0}); + std::sort(perm.begin(), perm.end(), + [&](size_t a, size_t b) { return indices[a] < indices[b]; }); + std::vector sorted_indices(n); + std::vector sorted_values(n * value_byte_size); + for (size_t i = 0; i < n; ++i) { + sorted_indices[i] = indices[perm[i]]; + std::memcpy(sorted_values.data() + i * value_byte_size, + values + perm[i] * value_byte_size, value_byte_size); + } + std::memcpy(indices, sorted_indices.data(), n * sizeof(uint32_t)); + std::memcpy(values, sorted_values.data(), n * value_byte_size); + for (size_t i = 1; i < n; ++i) { + if (indices[i] == indices[i - 1]) { + return true; + } + } + return false; +} + +} // namespace + void Doc::write_to_buffer(std::vector &buffer, const void *src, size_t size) { @@ -693,8 +761,8 @@ Doc::Ptr Doc::deserialize(const uint8_t *data, size_t /*size*/) { return doc; } -Status Doc::validate(const CollectionSchema::Ptr &schema, - bool is_update) const { +Status Doc::validate_and_sanitize(const CollectionSchema::Ptr &schema, + bool is_update) { if (!schema) { return Status::InternalError("schema is null during doc validation"); } @@ -739,7 +807,7 @@ Status Doc::validate(const CollectionSchema::Ptr &schema, } } - const Value &field_value = field_pair->second; + Value &field_value = field_pair->second; DataType expected_type = field_schema->data_type(); bool type_match = true; uint32_t value_dimension = 0; @@ -860,7 +928,7 @@ Status Doc::validate(const CollectionSchema::Ptr &schema, std::pair, std::vector>>( field_value); if (type_match) { - auto [sparse_indices, sparse_values] = std::get< + auto &[sparse_indices, sparse_values] = std::get< std::pair, std::vector>>( field_value); if (sparse_values.size() != sparse_indices.size()) { @@ -874,6 +942,14 @@ Status Doc::validate(const CollectionSchema::Ptr &schema, "] exceeds the maximum number of sparse indices (", kSparseMaxDimSize, ")"); } + if (sort_and_find_duplicates( + sparse_indices.data(), + reinterpret_cast(sparse_values.data()), + sparse_indices.size(), sizeof(float16_t))) { + return Status::InvalidArgument( + "Invalid doc[", pk_, "]: sparse vector field[", field_name, + "] contains duplicate indices"); + } } break; } @@ -895,6 +971,14 @@ Status Doc::validate(const CollectionSchema::Ptr &schema, "] exceeds the maximum number of sparse indices (", kSparseMaxDimSize, ")"); } + if (sort_and_find_duplicates( + sparse_indices.data(), + reinterpret_cast(sparse_values.data()), + sparse_indices.size(), sizeof(float))) { + return Status::InvalidArgument( + "Invalid doc[", pk_, "]: sparse vector field[", field_name, + "] contains duplicate indices"); + } } break; } @@ -1036,24 +1120,6 @@ size_t Doc::memory_usage() const { return usage; } -template -std::string vec_to_string(const std::vector &v) { - std::ostringstream oss; - oss << "["; - for (size_t i = 0; i < v.size(); ++i) { - if (i > 0) oss << ", "; - oss << +v[i]; // + from print as char - } - oss << "]"; - return oss.str(); -} - -template -struct overloaded : Ts... { - using Ts::operator()...; -}; -template -overloaded(Ts...) -> overloaded; std::string Doc::to_detail_string() const { std::stringstream oss; @@ -1202,7 +1268,7 @@ bool Doc::operator==(const Doc &other) const { return true; } -Status VectorQuery::validate(const FieldSchema *schema) const { +Status VectorQuery::validate_and_sanitize(const FieldSchema *schema) { if ((uint32_t)topk_ > kMaxQueryTopk) { return Status::InvalidArgument("Invalid query: topk[", topk_, "] exceeds the maximum allowed value of ", @@ -1274,12 +1340,40 @@ Status VectorQuery::validate(const FieldSchema *schema) const { "] is not a dense vector field"); } } else if (schema->is_sparse_vector()) { - // Validate sparse indices size - if (query_sparse_indices_.size() > kSparseMaxDimSize * sizeof(uint32_t)) { + size_t value_byte_size = 0; + switch (schema->data_type()) { + case DataType::SPARSE_VECTOR_FP32: + value_byte_size = sizeof(float); + break; + case DataType::SPARSE_VECTOR_FP16: + value_byte_size = sizeof(float16_t); + break; + default: + return Status::InvalidArgument( + "Invalid query: sparse vector type of field[", field_name_, + "] is not supported"); + } + if (query_sparse_indices_.size() % sizeof(uint32_t) != 0 || + query_sparse_values_.size() % value_byte_size != 0 || + query_sparse_indices_.size() / sizeof(uint32_t) != + query_sparse_values_.size() / value_byte_size) { + return Status::InvalidArgument( + "Invalid query: sparse vector query for field[", field_name_, + "] has mismatched indices and values sizes"); + } + size_t n_indices = query_sparse_indices_.size() / sizeof(uint32_t); + if (n_indices > kSparseMaxDimSize) { return Status::InvalidArgument( "Invalid query: too many sparse indices, the maximum allowed is ", kSparseMaxDimSize); } + if (sort_and_find_duplicates( + reinterpret_cast(query_sparse_indices_.data()), + query_sparse_values_.data(), n_indices, value_byte_size)) { + return Status::InvalidArgument( + "Invalid query: sparse vector query for field[", field_name_, + "] contains duplicate indices"); + } } else { return Status::InvalidArgument("Invalid query: field[", field_name_, "] is not a vector field"); diff --git a/src/include/zvec/c_api.h b/src/include/zvec/c_api.h index af21729..cbb174e 100644 --- a/src/include/zvec/c_api.h +++ b/src/include/zvec/c_api.h @@ -3044,9 +3044,9 @@ ZVEC_EXPORT size_t ZVEC_CALL zvec_doc_memory_usage(const zvec_doc_t *doc); * @param[out] error_msg Error message (needs manual release) * @return zvec_error_code_t Error code */ -ZVEC_EXPORT zvec_error_code_t ZVEC_CALL -zvec_doc_validate(const zvec_doc_t *doc, const zvec_collection_schema_t *schema, - bool is_update, char **error_msg); +ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_doc_validate_and_sanitize( + zvec_doc_t *doc, const zvec_collection_schema_t *schema, bool is_update, + char **error_msg); /** * @brief Get detailed string representation of document diff --git a/src/include/zvec/db/doc.h b/src/include/zvec/db/doc.h index c78028c..f702a43 100644 --- a/src/include/zvec/db/doc.h +++ b/src/include/zvec/db/doc.h @@ -260,8 +260,8 @@ class Doc { fields_.erase(field_name); } - Status validate(const CollectionSchema::Ptr &schema, - bool is_update = false) const; + Status validate_and_sanitize(const CollectionSchema::Ptr &schema, + bool is_update = false); size_t memory_usage() const; @@ -378,7 +378,7 @@ struct VectorQuery { std::optional> output_fields_; QueryParams::Ptr query_params_; - Status validate(const FieldSchema *schema) const; + Status validate_and_sanitize(const FieldSchema *schema); }; struct GroupByVectorQuery { diff --git a/tests/c/c_api_test.c b/tests/c/c_api_test.c index 8d66329..4955dce 100644 --- a/tests/c/c_api_test.c +++ b/tests/c/c_api_test.c @@ -4784,7 +4784,8 @@ void test_doc_advanced_functions(void) { &(int32_t){42}, sizeof(int32_t)); char *error_msg = NULL; - zvec_error_code_t err = zvec_doc_validate(val_doc, schema, false, &error_msg); + zvec_error_code_t err = + zvec_doc_validate_and_sanitize(val_doc, schema, false, &error_msg); TEST_ASSERT(err == ZVEC_OK); if (error_msg) { zvec_free(error_msg); diff --git a/tests/db/index/common/doc_test.cc b/tests/db/index/common/doc_test.cc index 986d766..5431411 100644 --- a/tests/db/index/common/doc_test.cc +++ b/tests/db/index/common/doc_test.cc @@ -531,65 +531,66 @@ TEST_F(DocDetailedTest, MixedDataTypes) { EXPECT_EQ(deserialized_sparse.second, sparse_vec.second); } -// Test doc validate with schema -TEST_F(DocDetailedTest, Validate) { - // test schema nullable=false, but doc's field is null +// Test doc validation and sanitization +TEST_F(DocDetailedTest, ValidateAndSanitization) { + // nullable=false: a doc with a null field is rejected { auto schema = test::TestHelper::CreateNormalSchema(false); auto doc = test::TestHelper::CreateDoc(1, *schema); - auto s = doc.validate(schema); + auto s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); doc = test::TestHelper::CreateDocNull(1, *schema); - s = doc.validate(schema); + s = doc.validate_and_sanitize(schema); ASSERT_FALSE(s.ok()); ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } + // nullable=true: a doc with a null field is accepted { auto schema = test::TestHelper::CreateNormalSchema(true); auto doc = test::TestHelper::CreateDoc(1, *schema); - auto s = doc.validate(schema); + auto s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); doc = test::TestHelper::CreateDocNull(1, *schema); - s = doc.validate(schema); + s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); } - // doc contained another field which not contained in schema + // doc has a field that is not declared in the schema { auto schema = test::TestHelper::CreateNormalSchema(false); auto doc = test::TestHelper::CreateDoc(1, *schema); - auto s = doc.validate(schema); + auto s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); doc.set("another_field", 1); - s = doc.validate(schema); + s = doc.validate_and_sanitize(schema); ASSERT_FALSE(s.ok()); ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // doc contained a mismatch scalar field + // scalar field value type does not match the schema { auto schema = test::TestHelper::CreateNormalSchema(false); auto doc = test::TestHelper::CreateDoc(1, *schema); - auto s = doc.validate(schema); + auto s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); doc.set("int32", std::string("1")); - s = doc.validate(schema); + s = doc.validate_and_sanitize(schema); ASSERT_FALSE(s.ok()); ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // doc contained a mismatch type vector field + // dense vector field element type does not match the schema { auto schema = test::TestHelper::CreateNormalSchema(false); auto doc = test::TestHelper::CreateDoc(1, *schema); - auto s = doc.validate(schema); + auto s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); std::string field = "dense_fp32"; @@ -597,16 +598,16 @@ TEST_F(DocDetailedTest, Validate) { ASSERT_NE(field_schema, nullptr); doc.set(field, std::vector(field_schema->dimension(), 1)); - s = doc.validate(schema); + s = doc.validate_and_sanitize(schema); ASSERT_FALSE(s.ok()); ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // doc contained a vector field with invalid dimension + // dense vector dimension does not match the schema { auto schema = test::TestHelper::CreateNormalSchema(false); auto doc = test::TestHelper::CreateDoc(1, *schema); - auto s = doc.validate(schema); + auto s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); std::string field = "dense_fp32"; @@ -614,21 +615,21 @@ TEST_F(DocDetailedTest, Validate) { ASSERT_NE(field_schema, nullptr); doc.set(field, std::vector(field_schema->dimension() - 1, 1.0)); - s = doc.validate(schema); + s = doc.validate_and_sanitize(schema); ASSERT_FALSE(s.ok()); ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); doc.set(field, std::vector()); - s = doc.validate(schema); + s = doc.validate_and_sanitize(schema); ASSERT_FALSE(s.ok()); ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // doc contained a sparse vector field with mismatch type + // sparse vector field value type does not match the schema { auto schema = test::TestHelper::CreateNormalSchema(false); auto doc = test::TestHelper::CreateDoc(1, *schema); - auto s = doc.validate(schema); + auto s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); std::string field = "sparse_fp32"; @@ -636,16 +637,16 @@ TEST_F(DocDetailedTest, Validate) { ASSERT_NE(field_schema, nullptr); doc.set(field, std::vector(field_schema->dimension(), 1)); - s = doc.validate(schema); + s = doc.validate_and_sanitize(schema); ASSERT_FALSE(s.ok()); ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // doc contained a sparse vector field with indices/values size mismatch + // sparse vector indices and values have different lengths { auto schema = test::TestHelper::CreateNormalSchema(false); auto doc = test::TestHelper::CreateDoc(1, *schema); - auto s = doc.validate(schema); + auto s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); std::string field = "sparse_fp32"; @@ -663,31 +664,64 @@ TEST_F(DocDetailedTest, Validate) { indices, values}; doc.set, std::vector>>( field, sparse_float_vec); - s = doc.validate(schema); + s = doc.validate_and_sanitize(schema); ASSERT_FALSE(s.ok()); ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // doc validate error + // sparse vector indices are sorted in place; duplicates are rejected { - Doc doc; - // schema is null - auto s = doc.validate(nullptr); - EXPECT_EQ(s.code(), StatusCode::INTERNAL_ERROR); - - // pk is null auto schema = test::TestHelper::CreateNormalSchema(false); - s = doc.validate(schema); - EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); + auto doc = test::TestHelper::CreateDoc(1, *schema); - // field type is undefined - schema->add_field( - std::make_shared("undefined", DataType::UNDEFINED, true)); - s = doc.validate(schema); - EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); + // unsorted indices are accepted and sorted in place + std::pair, std::vector> unsorted{ + {42u, 7u, 1000u, 3u, 128u, 17u, 99u}, + {0.7f, 0.1f, 0.9f, 0.2f, 0.5f, 0.3f, 0.6f}}; + doc.set, std::vector>>("sparse_fp32", + unsorted); + auto s = doc.validate_and_sanitize(schema); + ASSERT_TRUE(s.ok()) << s.message(); + const auto sorted_opt = + doc.get, std::vector>>( + "sparse_fp32"); + ASSERT_TRUE(sorted_opt.has_value()); + const std::vector expected_sorted_indices{3u, 7u, 17u, 42u, + 99u, 128u, 1000u}; + ASSERT_EQ(sorted_opt->first, expected_sorted_indices); + ASSERT_EQ(sorted_opt->second.size(), expected_sorted_indices.size()); + + // sorted indices with a duplicate are rejected + std::pair, std::vector> dup{ + {3u, 7u, 17u, 42u, 42u, 99u, 128u}, + {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f}}; + doc.set, std::vector>>("sparse_fp32", + dup); + s = doc.validate_and_sanitize(schema); + ASSERT_FALSE(s.ok()); + ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // doc validate more data type + // validate rejects: null schema, missing pk, undefined field type + { + Doc doc; + // null schema + auto s = doc.validate_and_sanitize(nullptr); + ASSERT_EQ(s.code(), StatusCode::INTERNAL_ERROR); + + // doc has no pk field + auto schema = test::TestHelper::CreateNormalSchema(false); + s = doc.validate_and_sanitize(schema); + ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); + + // schema contains a field with an undefined data type + schema->add_field( + std::make_shared("undefined", DataType::UNDEFINED, true)); + s = doc.validate_and_sanitize(schema); + ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); + } + + // validate accepts every supported field data type { auto schema = test::TestHelper::CreateNormalSchema(false); schema->add_field( @@ -731,10 +765,11 @@ TEST_F(DocDetailedTest, Validate) { auto doc = test::TestHelper::CreateDoc(1, *schema); - auto s = doc.validate(schema); + auto s = doc.validate_and_sanitize(schema); ASSERT_TRUE(s.ok()); } - // doc validate pk + + // pk with characters inside the allowed set is accepted { auto schema = test::TestHelper::CreateNormalSchema(false); std::vector valid_names = { @@ -777,10 +812,13 @@ TEST_F(DocDetailedTest, Validate) { }; for (auto pk : valid_names) { auto doc = test::TestHelper::CreateDoc(1, *schema, pk); - auto s = doc.validate(schema); - ASSERT_TRUE(s.ok()); + auto s = doc.validate_and_sanitize(schema); + ASSERT_TRUE(s.ok()) << "expected valid pk: " << pk + << ", got: " << s.message(); } } + + // pk that is too long or uses disallowed characters is rejected { auto schema = test::TestHelper::CreateNormalSchema(false); std::vector invalid_names = { @@ -817,9 +855,8 @@ TEST_F(DocDetailedTest, Validate) { }; for (auto pk : invalid_names) { auto doc = test::TestHelper::CreateDoc(1, *schema, pk); - auto s = doc.validate(schema); - if (s.ok()) std::cout << "pk:" << pk << std::endl; - ASSERT_FALSE(s.ok()); + auto s = doc.validate_and_sanitize(schema); + ASSERT_FALSE(s.ok()) << "expected invalid pk: " << pk; } } } @@ -1182,17 +1219,17 @@ TEST_F(DocDetailedTest, EqualityOperatorCoverage) { } -TEST(VectorQuery, Validate) { - // field schema is null when query without vector +TEST(VectorQuery, ValidateAndSanitize) { + // scalar-only query (no query vector): field schema is null { VectorQuery query; query.topk_ = 10; query.field_name_ = "field_name"; - auto s = query.validate(nullptr); + auto s = query.validate_and_sanitize(nullptr); EXPECT_TRUE(s.ok()); } - // field schema is null when query without vector + // vector query requires a non-null field schema { VectorQuery query; query.topk_ = 10; @@ -1202,34 +1239,24 @@ TEST(VectorQuery, Validate) { std::string(reinterpret_cast(query_vector.data()), query_vector.size() * sizeof(float)); query.query_vector_ = query_vector_str; - auto s = query.validate(nullptr); + auto s = query.validate_and_sanitize(nullptr); EXPECT_FALSE(s.ok()); EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // vector_query exceed topk - { - VectorQuery query; - query.field_name_ = "field_name"; - query.topk_ = 1000; - FieldSchema schema = - FieldSchema("field_name", DataType::VECTOR_FP32, 128, true); - auto s = query.validate(&schema); - EXPECT_FALSE(s.ok()); - EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); - } - // vector_query output_fields size exceed + + // output_fields count exceeds the allowed maximum { VectorQuery query; query.field_name_ = "field_name"; query.topk_ = 10; query.output_fields_ = std::vector(1025); FieldSchema schema = FieldSchema("field_name", DataType::INT32); - auto s = query.validate(&schema); + auto s = query.validate_and_sanitize(&schema); EXPECT_FALSE(s.ok()); EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // validate dense vector dimension + // dense vector query dimension must match the field schema { VectorQuery query; query.field_name_ = "field_name"; @@ -1241,37 +1268,122 @@ TEST(VectorQuery, Validate) { query.query_vector_ = query_vector_str; FieldSchema schema = FieldSchema("field_name", DataType::VECTOR_FP32, 4, true); - auto s = query.validate(&schema); + auto s = query.validate_and_sanitize(&schema); EXPECT_TRUE(s.ok()); query.query_vector_ = query_vector_str.substr(0, 3); - s = query.validate(&schema); + s = query.validate_and_sanitize(&schema); EXPECT_FALSE(s.ok()); EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); } - // validate sparse indices + // sparse query indices count must not exceed the allowed maximum { VectorQuery query; query.field_name_ = "field_name"; query.topk_ = 100; - std::vector query_indices = std::vector(16385); - std::string query_indices_str = + std::vector query_indices(16385); + std::vector query_values(16385); + query.query_sparse_indices_ = std::string(reinterpret_cast(query_indices.data()), query_indices.size() * sizeof(uint32_t)); - query.query_sparse_indices_ = query_indices_str; + query.query_sparse_values_ = + std::string(reinterpret_cast(query_values.data()), + query_values.size() * sizeof(float)); FieldSchema schema = FieldSchema("field_name", DataType::SPARSE_VECTOR_FP32); - auto s = query.validate(&schema); + auto s = query.validate_and_sanitize(&schema); EXPECT_FALSE(s.ok()); EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); - query.query_sparse_indices_ = query_indices_str.substr(0, 3); - s = query.validate(&schema); + // one valid index and matching value: accepted + uint32_t one_index = 0; + float one_value = 0.0f; + query.query_sparse_indices_ = + std::string(reinterpret_cast(&one_index), sizeof(uint32_t)); + query.query_sparse_values_ = + std::string(reinterpret_cast(&one_value), sizeof(float)); + s = query.validate_and_sanitize(&schema); EXPECT_TRUE(s.ok()); } - // validate query_params type matches index type + // sparse query must have matching counts, and indices must be strictly + // ascending and unique + { + auto pack_idx = [](const std::vector &v) { + return std::string(reinterpret_cast(v.data()), + v.size() * sizeof(uint32_t)); + }; + auto pack_val = [](const std::vector &v) { + return std::string(reinterpret_cast(v.data()), + v.size() * sizeof(float)); + }; + auto decode_idx = [](const std::string &buf) { + const auto *p = reinterpret_cast(buf.data()); + return std::vector(p, p + buf.size() / sizeof(uint32_t)); + }; + auto decode_val = [](const std::string &buf) { + const auto *p = reinterpret_cast(buf.data()); + return std::vector(p, p + buf.size() / sizeof(float)); + }; + FieldSchema schema = + FieldSchema("field_name", DataType::SPARSE_VECTOR_FP32); + + // unsorted indices are sorted in place + { + VectorQuery query; + query.field_name_ = "field_name"; + query.topk_ = 100; + query.query_sparse_indices_ = pack_idx({42u, 7u, 128u, 3u, 99u}); + query.query_sparse_values_ = pack_val({0.1f, 0.2f, 0.3f, 0.4f, 0.5f}); + auto s = query.validate_and_sanitize(&schema); + EXPECT_TRUE(s.ok()) << s.message(); + EXPECT_EQ(decode_idx(query.query_sparse_indices_), + (std::vector{3u, 7u, 42u, 99u, 128u})); + EXPECT_EQ(decode_val(query.query_sparse_values_), + (std::vector{0.4f, 0.2f, 0.1f, 0.5f, 0.3f})); + } + + // duplicates are rejected + { + VectorQuery query; + query.field_name_ = "field_name"; + query.topk_ = 100; + query.query_sparse_indices_ = pack_idx({3u, 7u, 42u, 42u, 99u}); + query.query_sparse_values_ = pack_val({0.1f, 0.2f, 0.3f, 0.4f, 0.5f}); + auto s = query.validate_and_sanitize(&schema); + EXPECT_FALSE(s.ok()); + EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); + + query.query_sparse_indices_ = pack_idx({42u, 3u, 7u, 42u, 99u}); + query.query_sparse_values_ = pack_val({0.1f, 0.2f, 0.3f, 0.4f, 0.5f}); + s = query.validate_and_sanitize(&schema); + EXPECT_FALSE(s.ok()); + EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); + } + + // mismatched counts are rejected + { + VectorQuery query; + query.field_name_ = "field_name"; + query.topk_ = 100; + const auto idx_before = pack_idx({3u, 2u, 1u, 4u}); + const auto val_before = pack_val({0.1f, 0.2f, 0.3f, 0.4f}); + query.query_sparse_indices_ = idx_before; + query.query_sparse_values_ = val_before; + auto s = query.validate_and_sanitize(&schema); + EXPECT_TRUE(s.ok()) << s.message(); + EXPECT_EQ(query.query_sparse_indices_, pack_idx({1u, 2u, 3u, 4u})); + EXPECT_EQ(query.query_sparse_values_, pack_val({0.3f, 0.2f, 0.1f, 0.4f})); + + query.query_sparse_values_ = pack_val({0.1f, 0.2f, 0.3f}); + s = query.validate_and_sanitize(&schema); + EXPECT_FALSE(s.ok()); + EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); + } + } + + // query_params type must match the field's index type { VectorQuery query; query.field_name_ = "embedding"; @@ -1285,16 +1397,16 @@ TEST(VectorQuery, Validate) { std::make_shared(MetricType::L2)); query.query_params_ = std::make_shared(150); - auto s = query.validate(&schema); + auto s = query.validate_and_sanitize(&schema); EXPECT_TRUE(s.ok()); query.query_params_ = std::make_shared(50); - s = query.validate(&schema); + s = query.validate_and_sanitize(&schema); EXPECT_FALSE(s.ok()); EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT); query.query_params_ = nullptr; - s = query.validate(&schema); + s = query.validate_and_sanitize(&schema); EXPECT_TRUE(s.ok()); } }