xiaoxue_admin
/
zvec
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: sparse vector indices should be ordered (#382)

This commit is contained in:
Qinren Zhou 2026-05-07 16:40:49 +08:00 committed by GitHub
parent d02ae2b74e
commit 68a497efdb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 329 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/binding/c/c_api.cc
View File

@ -4307,7 +4307,7 @@ size_t zvec_doc_memory_usage(const zvec_doc_t *doc) {
return doc_ptr->memory_usage();)
}
zvec_error_code_t zvec_doc_validate(const zvec_doc_t *doc,
zvec_error_code_t zvec_doc_validate_and_sanitize(zvec_doc_t *doc,
const zvec_collection_schema_t *schema,
bool is_update, char **error_msg) {
if (!doc || !schema) {
@ -4327,15 +4327,15 @@ zvec_error_code_t zvec_doc_validate(const zvec_doc_t *doc,
return status_to_error_code(status);
}
auto doc_ptr = reinterpret_cast<const zvec::Doc *>(doc);
status = doc_ptr->validate(schema_ptr, is_update); if (!status.ok()) {
auto doc_ptr = reinterpret_cast<zvec::Doc *>(doc);
status = doc_ptr->validate_and_sanitize(schema_ptr, is_update); if (!status.ok()) {
if (error_msg) {
*error_msg = copy_string(status.message());
}
return status_to_error_code(status);
}
if (error_msg) { *error_msg = nullptr; }
if (error_msg) { *error_msg = nullptr; }
return ZVEC_OK;)
}

12
src/db/collection.cc
View File

@ -35,7 +35,6 @@
#include "db/common/file_helper.h"
#include "db/common/profiler.h"
#include "db/common/typedef.h"
#include "db/index/column/vector_column/vector_column_indexer.h"
#include "db/index/common/delete_store.h"
#include "db/index/common/id_map.h"
#include "db/index/common/index_filter.h"
@ -1443,8 +1442,8 @@ Result<WriteResults> CollectionImpl::write_impl(std::vector<Doc> &docs,
CHECK_DESTROY_RETURN_STATUS_EXPECTED(destroyed_, false);
for (auto &&doc : docs) {
auto validate = doc.validate(schema_, mode == WriteMode::UPDATE);
CHECK_RETURN_STATUS_EXPECTED(validate);
auto s = doc.validate_and_sanitize(schema_, mode == WriteMode::UPDATE);
CHECK_RETURN_STATUS_EXPECTED(s);
}
// TODO: The granularity of the write_lock is too coarse.
@ -1458,7 +1457,6 @@ Result<WriteResults> CollectionImpl::write_impl(std::vector<Doc> &docs,
kMaxWriteBatchSize));
}
// validate docs
for (auto &&doc : docs) {
if (need_switch_to_new_segment()) {
auto s = switch_to_new_segment_for_writing();
@ -1583,7 +1581,9 @@ Result<DocPtrList> CollectionImpl::Query(const VectorQuery &query) const {
CHECK_DESTROY_RETURN_STATUS_EXPECTED(destroyed_, false);
auto s = query.validate(schema_->get_vector_field(query.field_name_));
VectorQuery sanitized = query;
auto s = sanitized.validate_and_sanitize(
schema_->get_vector_field(sanitized.field_name_));
CHECK_RETURN_STATUS_EXPECTED(s);
auto segments = get_all_segments();
@ -1591,7 +1591,7 @@ Result<DocPtrList> CollectionImpl::Query(const VectorQuery &query) const {
return DocPtrList();
}
return sql_engine_->execute(schema_, query, segments);
return sql_engine_->execute(schema_, sanitized, segments);
}
Result<GroupResults> CollectionImpl::GroupByQuery(

144
src/db/index/common/doc.cc
View File

@ -11,10 +11,13 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <numeric>
#include <regex>
#include <stdexcept>
#include <zvec/ailego/internal/platform.h>
@ -114,6 +117,9 @@ std::string get_value_type_name(const Doc::Value &value, bool is_vector) {
value);
}
namespace {
template <typename T>
T byte_swap(T value) {
if constexpr (std::is_same_v<T, float16_t>) {
@ -159,6 +165,68 @@ T read_value_from_buffer(const uint8_t *&data) {
return value;
}
template <typename T>
std::string vec_to_string(const std::vector<T> &v) {
std::ostringstream oss;
oss << "[";
for (size_t i = 0; i < v.size(); ++i) {
if (i > 0) oss << ", ";
oss << +v[i]; // + from print as char
}
oss << "]";
return oss.str();
}
template <class... Ts>
struct overloaded : Ts... {
using Ts::operator()...;
};
template <class... Ts>
overloaded(Ts...) -> overloaded<Ts...>;
bool sort_and_find_duplicates(uint32_t *indices, char *values, size_t n,
size_t value_byte_size) {
if (n <= 1) {
return false;
}
bool already_sorted = true;
for (size_t i = 1; i < n; ++i) {
if (indices[i] == indices[i - 1]) {
return true;
}
if (indices[i] < indices[i - 1]) {
already_sorted = false;
break;
}
}
if (already_sorted) {
return false;
}
std::vector<size_t> perm(n);
std::iota(perm.begin(), perm.end(), size_t{0});
std::sort(perm.begin(), perm.end(),
[&](size_t a, size_t b) { return indices[a] < indices[b]; });
std::vector<uint32_t> sorted_indices(n);
std::vector<char> sorted_values(n * value_byte_size);
for (size_t i = 0; i < n; ++i) {
sorted_indices[i] = indices[perm[i]];
std::memcpy(sorted_values.data() + i * value_byte_size,
values + perm[i] * value_byte_size, value_byte_size);
}
std::memcpy(indices, sorted_indices.data(), n * sizeof(uint32_t));
std::memcpy(values, sorted_values.data(), n * value_byte_size);
for (size_t i = 1; i < n; ++i) {
if (indices[i] == indices[i - 1]) {
return true;
}
}
return false;
}
} // namespace
void Doc::write_to_buffer(std::vector<uint8_t> &buffer, const void *src,
size_t size) {
@ -693,8 +761,8 @@ Doc::Ptr Doc::deserialize(const uint8_t *data, size_t /*size*/) {
return doc;
}
Status Doc::validate(const CollectionSchema::Ptr &schema,
bool is_update) const {
Status Doc::validate_and_sanitize(const CollectionSchema::Ptr &schema,
bool is_update) {
if (!schema) {
return Status::InternalError("schema is null during doc validation");
}
@ -739,7 +807,7 @@ Status Doc::validate(const CollectionSchema::Ptr &schema,
}
}
const Value &field_value = field_pair->second;
Value &field_value = field_pair->second;
DataType expected_type = field_schema->data_type();
bool type_match = true;
uint32_t value_dimension = 0;
@ -860,7 +928,7 @@ Status Doc::validate(const CollectionSchema::Ptr &schema,
std::pair<std::vector<uint32_t>, std::vector<float16_t>>>(
field_value);
if (type_match) {
auto [sparse_indices, sparse_values] = std::get<
auto &[sparse_indices, sparse_values] = std::get<
std::pair<std::vector<uint32_t>, std::vector<float16_t>>>(
field_value);
if (sparse_values.size() != sparse_indices.size()) {
@ -874,6 +942,14 @@ Status Doc::validate(const CollectionSchema::Ptr &schema,
"] exceeds the maximum number of sparse indices (",
kSparseMaxDimSize, ")");
}
if (sort_and_find_duplicates(
sparse_indices.data(),
reinterpret_cast<char *>(sparse_values.data()),
sparse_indices.size(), sizeof(float16_t))) {
return Status::InvalidArgument(
"Invalid doc[", pk_, "]: sparse vector field[", field_name,
"] contains duplicate indices");
}
}
break;
}
@ -895,6 +971,14 @@ Status Doc::validate(const CollectionSchema::Ptr &schema,
"] exceeds the maximum number of sparse indices (",
kSparseMaxDimSize, ")");
}
if (sort_and_find_duplicates(
sparse_indices.data(),
reinterpret_cast<char *>(sparse_values.data()),
sparse_indices.size(), sizeof(float))) {
return Status::InvalidArgument(
"Invalid doc[", pk_, "]: sparse vector field[", field_name,
"] contains duplicate indices");
}
}
break;
}
@ -1036,24 +1120,6 @@ size_t Doc::memory_usage() const {
return usage;
}
template <typename T>
std::string vec_to_string(const std::vector<T> &v) {
std::ostringstream oss;
oss << "[";
for (size_t i = 0; i < v.size(); ++i) {
if (i > 0) oss << ", ";
oss << +v[i]; // + from print as char
}
oss << "]";
return oss.str();
}
template <class... Ts>
struct overloaded : Ts... {
using Ts::operator()...;
};
template <class... Ts>
overloaded(Ts...) -> overloaded<Ts...>;
std::string Doc::to_detail_string() const {
std::stringstream oss;
@ -1202,7 +1268,7 @@ bool Doc::operator==(const Doc &other) const {
return true;
}
Status VectorQuery::validate(const FieldSchema *schema) const {
Status VectorQuery::validate_and_sanitize(const FieldSchema *schema) {
if ((uint32_t)topk_ > kMaxQueryTopk) {
return Status::InvalidArgument("Invalid query: topk[", topk_,
"] exceeds the maximum allowed value of ",
@ -1274,12 +1340,40 @@ Status VectorQuery::validate(const FieldSchema *schema) const {
"] is not a dense vector field");
}
} else if (schema->is_sparse_vector()) {
// Validate sparse indices size
if (query_sparse_indices_.size() > kSparseMaxDimSize * sizeof(uint32_t)) {
size_t value_byte_size = 0;
switch (schema->data_type()) {
case DataType::SPARSE_VECTOR_FP32:
value_byte_size = sizeof(float);
break;
case DataType::SPARSE_VECTOR_FP16:
value_byte_size = sizeof(float16_t);
break;
default:
return Status::InvalidArgument(
"Invalid query: sparse vector type of field[", field_name_,
"] is not supported");
}
if (query_sparse_indices_.size() % sizeof(uint32_t) != 0 ||
query_sparse_values_.size() % value_byte_size != 0 ||
query_sparse_indices_.size() / sizeof(uint32_t) !=
query_sparse_values_.size() / value_byte_size) {
return Status::InvalidArgument(
"Invalid query: sparse vector query for field[", field_name_,
"] has mismatched indices and values sizes");
}
size_t n_indices = query_sparse_indices_.size() / sizeof(uint32_t);
if (n_indices > kSparseMaxDimSize) {
return Status::InvalidArgument(
"Invalid query: too many sparse indices, the maximum allowed is ",
kSparseMaxDimSize);
}
if (sort_and_find_duplicates(
reinterpret_cast<uint32_t *>(query_sparse_indices_.data()),
query_sparse_values_.data(), n_indices, value_byte_size)) {
return Status::InvalidArgument(
"Invalid query: sparse vector query for field[", field_name_,
"] contains duplicate indices");
}
} else {
return Status::InvalidArgument("Invalid query: field[", field_name_,
"] is not a vector field");

6
src/include/zvec/c_api.h
View File

@ -3044,9 +3044,9 @@ ZVEC_EXPORT size_t ZVEC_CALL zvec_doc_memory_usage(const zvec_doc_t *doc);
* @param[out] error_msg Error message (needs manual release)
* @return zvec_error_code_t Error code
*/
ZVEC_EXPORT zvec_error_code_t ZVEC_CALL
zvec_doc_validate(const zvec_doc_t *doc, const zvec_collection_schema_t *schema,
bool is_update, char **error_msg);
ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_doc_validate_and_sanitize(
zvec_doc_t *doc, const zvec_collection_schema_t *schema, bool is_update,
char **error_msg);
/**
* @brief Get detailed string representation of document

6
src/include/zvec/db/doc.h
View File

@ -260,8 +260,8 @@ class Doc {
fields_.erase(field_name);
}
Status validate(const CollectionSchema::Ptr &schema,
bool is_update = false) const;
Status validate_and_sanitize(const CollectionSchema::Ptr &schema,
bool is_update = false);
size_t memory_usage() const;
@ -378,7 +378,7 @@ struct VectorQuery {
std::optional<std::vector<std::string>> output_fields_;
QueryParams::Ptr query_params_;
Status validate(const FieldSchema *schema) const;
Status validate_and_sanitize(const FieldSchema *schema);
};
struct GroupByVectorQuery {

3
tests/c/c_api_test.c
View File

@ -4784,7 +4784,8 @@ void test_doc_advanced_functions(void) {
&(int32_t){42}, sizeof(int32_t));
char *error_msg = NULL;
zvec_error_code_t err = zvec_doc_validate(val_doc, schema, false, &error_msg);
zvec_error_code_t err =
zvec_doc_validate_and_sanitize(val_doc, schema, false, &error_msg);
TEST_ASSERT(err == ZVEC_OK);
if (error_msg) {
zvec_free(error_msg);

272
tests/db/index/common/doc_test.cc
View File

@ -531,65 +531,66 @@ TEST_F(DocDetailedTest, MixedDataTypes) {
EXPECT_EQ(deserialized_sparse.second, sparse_vec.second);
}
// Test doc validate with schema
TEST_F(DocDetailedTest, Validate) {
// test schema nullable=false, but doc's field is null
// Test doc validation and sanitization
TEST_F(DocDetailedTest, ValidateAndSanitization) {
// nullable=false: a doc with a null field is rejected
{
auto schema = test::TestHelper::CreateNormalSchema(false);
auto doc = test::TestHelper::CreateDoc(1, *schema);
auto s = doc.validate(schema);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
doc = test::TestHelper::CreateDocNull(1, *schema);
s = doc.validate(schema);
s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// nullable=true: a doc with a null field is accepted
{
auto schema = test::TestHelper::CreateNormalSchema(true);
auto doc = test::TestHelper::CreateDoc(1, *schema);
auto s = doc.validate(schema);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
doc = test::TestHelper::CreateDocNull(1, *schema);
s = doc.validate(schema);
s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
}
// doc contained another field which not contained in schema
// doc has a field that is not declared in the schema
{
auto schema = test::TestHelper::CreateNormalSchema(false);
auto doc = test::TestHelper::CreateDoc(1, *schema);
auto s = doc.validate(schema);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
doc.set("another_field", 1);
s = doc.validate(schema);
s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// doc contained a mismatch scalar field
// scalar field value type does not match the schema
{
auto schema = test::TestHelper::CreateNormalSchema(false);
auto doc = test::TestHelper::CreateDoc(1, *schema);
auto s = doc.validate(schema);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
doc.set("int32", std::string("1"));
s = doc.validate(schema);
s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// doc contained a mismatch type vector field
// dense vector field element type does not match the schema
{
auto schema = test::TestHelper::CreateNormalSchema(false);
auto doc = test::TestHelper::CreateDoc(1, *schema);
auto s = doc.validate(schema);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
std::string field = "dense_fp32";
@ -597,16 +598,16 @@ TEST_F(DocDetailedTest, Validate) {
ASSERT_NE(field_schema, nullptr);
doc.set(field, std::vector<int16_t>(field_schema->dimension(), 1));
s = doc.validate(schema);
s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// doc contained a vector field with invalid dimension
// dense vector dimension does not match the schema
{
auto schema = test::TestHelper::CreateNormalSchema(false);
auto doc = test::TestHelper::CreateDoc(1, *schema);
auto s = doc.validate(schema);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
std::string field = "dense_fp32";
@ -614,21 +615,21 @@ TEST_F(DocDetailedTest, Validate) {
ASSERT_NE(field_schema, nullptr);
doc.set(field, std::vector<float>(field_schema->dimension() - 1, 1.0));
s = doc.validate(schema);
s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
doc.set(field, std::vector<float>());
s = doc.validate(schema);
s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// doc contained a sparse vector field with mismatch type
// sparse vector field value type does not match the schema
{
auto schema = test::TestHelper::CreateNormalSchema(false);
auto doc = test::TestHelper::CreateDoc(1, *schema);
auto s = doc.validate(schema);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
std::string field = "sparse_fp32";
@ -636,16 +637,16 @@ TEST_F(DocDetailedTest, Validate) {
ASSERT_NE(field_schema, nullptr);
doc.set(field, std::vector<int16_t>(field_schema->dimension(), 1));
s = doc.validate(schema);
s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// doc contained a sparse vector field with indices/values size mismatch
// sparse vector indices and values have different lengths
{
auto schema = test::TestHelper::CreateNormalSchema(false);
auto doc = test::TestHelper::CreateDoc(1, *schema);
auto s = doc.validate(schema);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
std::string field = "sparse_fp32";
@ -663,31 +664,64 @@ TEST_F(DocDetailedTest, Validate) {
indices, values};
doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>(
field, sparse_float_vec);
s = doc.validate(schema);
s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// doc validate error
// sparse vector indices are sorted in place; duplicates are rejected
{
Doc doc;
// schema is null
auto s = doc.validate(nullptr);
EXPECT_EQ(s.code(), StatusCode::INTERNAL_ERROR);
// pk is null
auto schema = test::TestHelper::CreateNormalSchema(false);
s = doc.validate(schema);
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
auto doc = test::TestHelper::CreateDoc(1, *schema);
// field type is undefined
schema->add_field(
std::make_shared<FieldSchema>("undefined", DataType::UNDEFINED, true));
s = doc.validate(schema);
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
// unsorted indices are accepted and sorted in place
std::pair<std::vector<uint32_t>, std::vector<float>> unsorted{
{42u, 7u, 1000u, 3u, 128u, 17u, 99u},
{0.7f, 0.1f, 0.9f, 0.2f, 0.5f, 0.3f, 0.6f}};
doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>("sparse_fp32",
unsorted);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok()) << s.message();
const auto sorted_opt =
doc.get<std::pair<std::vector<uint32_t>, std::vector<float>>>(
"sparse_fp32");
ASSERT_TRUE(sorted_opt.has_value());
const std::vector<uint32_t> expected_sorted_indices{3u, 7u, 17u, 42u,
99u, 128u, 1000u};
ASSERT_EQ(sorted_opt->first, expected_sorted_indices);
ASSERT_EQ(sorted_opt->second.size(), expected_sorted_indices.size());
// sorted indices with a duplicate are rejected
std::pair<std::vector<uint32_t>, std::vector<float>> dup{
{3u, 7u, 17u, 42u, 42u, 99u, 128u},
{0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f}};
doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>("sparse_fp32",
dup);
s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok());
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// doc validate more data type
// validate rejects: null schema, missing pk, undefined field type
{
Doc doc;
// null schema
auto s = doc.validate_and_sanitize(nullptr);
ASSERT_EQ(s.code(), StatusCode::INTERNAL_ERROR);
// doc has no pk field
auto schema = test::TestHelper::CreateNormalSchema(false);
s = doc.validate_and_sanitize(schema);
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
// schema contains a field with an undefined data type
schema->add_field(
std::make_shared<FieldSchema>("undefined", DataType::UNDEFINED, true));
s = doc.validate_and_sanitize(schema);
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// validate accepts every supported field data type
{
auto schema = test::TestHelper::CreateNormalSchema(false);
schema->add_field(
@ -731,10 +765,11 @@ TEST_F(DocDetailedTest, Validate) {
auto doc = test::TestHelper::CreateDoc(1, *schema);
auto s = doc.validate(schema);
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok());
}
// doc validate pk
// pk with characters inside the allowed set is accepted
{
auto schema = test::TestHelper::CreateNormalSchema(false);
std::vector<std::string> valid_names = {
@ -777,10 +812,13 @@ TEST_F(DocDetailedTest, Validate) {
};
for (auto pk : valid_names) {
auto doc = test::TestHelper::CreateDoc(1, *schema, pk);
auto s = doc.validate(schema);
ASSERT_TRUE(s.ok());
auto s = doc.validate_and_sanitize(schema);
ASSERT_TRUE(s.ok()) << "expected valid pk: " << pk
<< ", got: " << s.message();
}
}
// pk that is too long or uses disallowed characters is rejected
{
auto schema = test::TestHelper::CreateNormalSchema(false);
std::vector<std::string> invalid_names = {
@ -817,9 +855,8 @@ TEST_F(DocDetailedTest, Validate) {
};
for (auto pk : invalid_names) {
auto doc = test::TestHelper::CreateDoc(1, *schema, pk);
auto s = doc.validate(schema);
if (s.ok()) std::cout << "pk:" << pk << std::endl;
ASSERT_FALSE(s.ok());
auto s = doc.validate_and_sanitize(schema);
ASSERT_FALSE(s.ok()) << "expected invalid pk: " << pk;
}
}
}
@ -1182,17 +1219,17 @@ TEST_F(DocDetailedTest, EqualityOperatorCoverage) {
}
TEST(VectorQuery, Validate) {
// field schema is null when query without vector
TEST(VectorQuery, ValidateAndSanitize) {
// scalar-only query (no query vector): field schema is null
{
VectorQuery query;
query.topk_ = 10;
query.field_name_ = "field_name";
auto s = query.validate(nullptr);
auto s = query.validate_and_sanitize(nullptr);
EXPECT_TRUE(s.ok());
}
// field schema is null when query without vector
// vector query requires a non-null field schema
{
VectorQuery query;
query.topk_ = 10;
@ -1202,34 +1239,24 @@ TEST(VectorQuery, Validate) {
std::string(reinterpret_cast<char *>(query_vector.data()),
query_vector.size() * sizeof(float));
query.query_vector_ = query_vector_str;
auto s = query.validate(nullptr);
auto s = query.validate_and_sanitize(nullptr);
EXPECT_FALSE(s.ok());
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// vector_query exceed topk
{
VectorQuery query;
query.field_name_ = "field_name";
query.topk_ = 1000;
FieldSchema schema =
FieldSchema("field_name", DataType::VECTOR_FP32, 128, true);
auto s = query.validate(&schema);
EXPECT_FALSE(s.ok());
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// vector_query output_fields size exceed
// output_fields count exceeds the allowed maximum
{
VectorQuery query;
query.field_name_ = "field_name";
query.topk_ = 10;
query.output_fields_ = std::vector<std::string>(1025);
FieldSchema schema = FieldSchema("field_name", DataType::INT32);
auto s = query.validate(&schema);
auto s = query.validate_and_sanitize(&schema);
EXPECT_FALSE(s.ok());
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// validate dense vector dimension
// dense vector query dimension must match the field schema
{
VectorQuery query;
query.field_name_ = "field_name";
@ -1241,37 +1268,122 @@ TEST(VectorQuery, Validate) {
query.query_vector_ = query_vector_str;
FieldSchema schema =
FieldSchema("field_name", DataType::VECTOR_FP32, 4, true);
auto s = query.validate(&schema);
auto s = query.validate_and_sanitize(&schema);
EXPECT_TRUE(s.ok());
query.query_vector_ = query_vector_str.substr(0, 3);
s = query.validate(&schema);
s = query.validate_and_sanitize(&schema);
EXPECT_FALSE(s.ok());
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// validate sparse indices
// sparse query indices count must not exceed the allowed maximum
{
VectorQuery query;
query.field_name_ = "field_name";
query.topk_ = 100;
std::vector<uint32_t> query_indices = std::vector<uint32_t>(16385);
std::string query_indices_str =
std::vector<uint32_t> query_indices(16385);
std::vector<float> query_values(16385);
query.query_sparse_indices_ =
std::string(reinterpret_cast<char *>(query_indices.data()),
query_indices.size() * sizeof(uint32_t));
query.query_sparse_indices_ = query_indices_str;
query.query_sparse_values_ =
std::string(reinterpret_cast<char *>(query_values.data()),
query_values.size() * sizeof(float));
FieldSchema schema =
FieldSchema("field_name", DataType::SPARSE_VECTOR_FP32);
auto s = query.validate(&schema);
auto s = query.validate_and_sanitize(&schema);
EXPECT_FALSE(s.ok());
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
query.query_sparse_indices_ = query_indices_str.substr(0, 3);
s = query.validate(&schema);
// one valid index and matching value: accepted
uint32_t one_index = 0;
float one_value = 0.0f;
query.query_sparse_indices_ =
std::string(reinterpret_cast<char *>(&one_index), sizeof(uint32_t));
query.query_sparse_values_ =
std::string(reinterpret_cast<char *>(&one_value), sizeof(float));
s = query.validate_and_sanitize(&schema);
EXPECT_TRUE(s.ok());
}
// validate query_params type matches index type
// sparse query must have matching counts, and indices must be strictly
// ascending and unique
{
auto pack_idx = [](const std::vector<uint32_t> &v) {
return std::string(reinterpret_cast<const char *>(v.data()),
v.size() * sizeof(uint32_t));
};
auto pack_val = [](const std::vector<float> &v) {
return std::string(reinterpret_cast<const char *>(v.data()),
v.size() * sizeof(float));
};
auto decode_idx = [](const std::string &buf) {
const auto *p = reinterpret_cast<const uint32_t *>(buf.data());
return std::vector<uint32_t>(p, p + buf.size() / sizeof(uint32_t));
};
auto decode_val = [](const std::string &buf) {
const auto *p = reinterpret_cast<const float *>(buf.data());
return std::vector<float>(p, p + buf.size() / sizeof(float));
};
FieldSchema schema =
FieldSchema("field_name", DataType::SPARSE_VECTOR_FP32);
// unsorted indices are sorted in place
{
VectorQuery query;
query.field_name_ = "field_name";
query.topk_ = 100;
query.query_sparse_indices_ = pack_idx({42u, 7u, 128u, 3u, 99u});
query.query_sparse_values_ = pack_val({0.1f, 0.2f, 0.3f, 0.4f, 0.5f});
auto s = query.validate_and_sanitize(&schema);
EXPECT_TRUE(s.ok()) << s.message();
EXPECT_EQ(decode_idx(query.query_sparse_indices_),
(std::vector<uint32_t>{3u, 7u, 42u, 99u, 128u}));
EXPECT_EQ(decode_val(query.query_sparse_values_),
(std::vector<float>{0.4f, 0.2f, 0.1f, 0.5f, 0.3f}));
}
// duplicates are rejected
{
VectorQuery query;
query.field_name_ = "field_name";
query.topk_ = 100;
query.query_sparse_indices_ = pack_idx({3u, 7u, 42u, 42u, 99u});
query.query_sparse_values_ = pack_val({0.1f, 0.2f, 0.3f, 0.4f, 0.5f});
auto s = query.validate_and_sanitize(&schema);
EXPECT_FALSE(s.ok());
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
query.query_sparse_indices_ = pack_idx({42u, 3u, 7u, 42u, 99u});
query.query_sparse_values_ = pack_val({0.1f, 0.2f, 0.3f, 0.4f, 0.5f});
s = query.validate_and_sanitize(&schema);
EXPECT_FALSE(s.ok());
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
// mismatched counts are rejected
{
VectorQuery query;
query.field_name_ = "field_name";
query.topk_ = 100;
const auto idx_before = pack_idx({3u, 2u, 1u, 4u});
const auto val_before = pack_val({0.1f, 0.2f, 0.3f, 0.4f});
query.query_sparse_indices_ = idx_before;
query.query_sparse_values_ = val_before;
auto s = query.validate_and_sanitize(&schema);
EXPECT_TRUE(s.ok()) << s.message();
EXPECT_EQ(query.query_sparse_indices_, pack_idx({1u, 2u, 3u, 4u}));
EXPECT_EQ(query.query_sparse_values_, pack_val({0.3f, 0.2f, 0.1f, 0.4f}));
query.query_sparse_values_ = pack_val({0.1f, 0.2f, 0.3f});
s = query.validate_and_sanitize(&schema);
EXPECT_FALSE(s.ok());
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
}
}
// query_params type must match the field's index type
{
VectorQuery query;
query.field_name_ = "embedding";
@ -1285,16 +1397,16 @@ TEST(VectorQuery, Validate) {
std::make_shared<HnswIndexParams>(MetricType::L2));
query.query_params_ = std::make_shared<HnswQueryParams>(150);
auto s = query.validate(&schema);
auto s = query.validate_and_sanitize(&schema);
EXPECT_TRUE(s.ok());
query.query_params_ = std::make_shared<IVFQueryParams>(50);
s = query.validate(&schema);
s = query.validate_and_sanitize(&schema);
EXPECT_FALSE(s.ok());
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
query.query_params_ = nullptr;
s = query.validate(&schema);
s = query.validate_and_sanitize(&schema);
EXPECT_TRUE(s.ok());
}
}