xiaoxue_admin
/
zvec
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: enlarge indice size limit for sparse vectors (#229)

This commit is contained in:
Qinren Zhou 2026-03-16 17:12:24 +08:00 committed by GitHub
parent 5124fbc072
commit 3c1241f7c6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 43 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/core/algorithm/flat_sparse/flat_sparse_streamer.cc
View File

@ -195,8 +195,10 @@ int FlatSparseStreamer::add_impl(uint64_t pkey, const uint32_t sparse_count,
}
if (ailego_unlikely(sparse_count > PARAM_FLAT_SPARSE_MAX_DIM_SIZE)) {
LOG_ERROR("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
sparse_count, (size_t)pkey);
LOG_ERROR(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), key=%zu",
sparse_count, PARAM_FLAT_SPARSE_MAX_DIM_SIZE, (size_t)pkey);
(*stats_.mutable_discarded_count())++;
return IndexError_InvalidValue;
}
@ -252,8 +254,10 @@ int FlatSparseStreamer::add_with_id_impl(uint32_t pkey,
}
if (ailego_unlikely(sparse_count > PARAM_FLAT_SPARSE_MAX_DIM_SIZE)) {
LOG_ERROR("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
sparse_count, (size_t)pkey);
LOG_ERROR(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), key=%zu",
sparse_count, PARAM_FLAT_SPARSE_MAX_DIM_SIZE, (size_t)pkey);
(*stats_.mutable_discarded_count())++;
return IndexError_InvalidValue;
}

2
src/core/algorithm/flat_sparse/flat_sparse_utility.h
View File

@ -19,7 +19,7 @@
namespace zvec {
namespace core {
static constexpr uint32_t PARAM_FLAT_SPARSE_MAX_DIM_SIZE = 4096;
static constexpr uint32_t PARAM_FLAT_SPARSE_MAX_DIM_SIZE = 16384;
static const std::string PARAM_FLAT_SPARSE_META_SEG_ID =
"bruteforce_sparse_meta";

8
src/core/algorithm/hnsw_sparse/hnsw_sparse_builder_entity.cc
View File

@ -88,9 +88,11 @@ int HnswSparseBuilderEntity::add_vector(level_t level, key_t key,
const uint32_t sparse_count,
const uint32_t *sparse_indices,
const void *sparse_vec, node_id_t *id) {
if (ailego_unlikely(sparse_count >= HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
sparse_count, (size_t)key);
if (ailego_unlikely(sparse_count > HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), key=%zu",
sparse_count, HnswSparseEntity::kSparseMaxDimSize, (size_t)key);
return IndexError_InvalidValue;
}

2
src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h
View File

@ -610,7 +610,7 @@ class HnswSparseEntity {
constexpr static uint32_t kSparseMetaSize = 2u * sizeof(uint64_t);
constexpr static float kDefaultSparseNeighborRatio = 0.5f;
constexpr static uint32_t kSparseMaxDimSize = 4096;
constexpr static uint32_t kSparseMaxDimSize = 16384;
constexpr static float kDefaultQueryFilteringRatio = 0.0f; // turn off
protected:

16
src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer.cc
View File

@ -438,9 +438,11 @@ int HnswSparseStreamer::add_with_id_impl(uint32_t id,
return ret;
}
if (ailego_unlikely(sparse_count >= HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN("Add vector failed, dim size too larg, dim_size=%u, id=%u",
sparse_count, id);
if (ailego_unlikely(sparse_count > HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), id=%u",
sparse_count, HnswSparseEntity::kSparseMaxDimSize, id);
return IndexError_InvalidValue;
}
@ -523,9 +525,11 @@ int HnswSparseStreamer::add_impl(uint64_t pkey, const uint32_t sparse_count,
return ret;
}
if (ailego_unlikely(sparse_count >= HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
sparse_count, (size_t)pkey);
if (ailego_unlikely(sparse_count > HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), key=%zu",
sparse_count, HnswSparseEntity::kSparseMaxDimSize, (size_t)pkey);
return IndexError_InvalidValue;
}

2
src/db/common/constants.h
View File

@ -32,7 +32,7 @@ const std::string GLOBAL_DOC_ID = "_zvec_g_doc_id_";
const std::string USER_ID = "_zvec_uid_";
const int kSparseMaxDimSize = 4096;
const int kSparseMaxDimSize = 16384;
const int64_t kMaxRecordBatchNumRows = 4096;

18
src/db/index/common/doc.cc
View File

@ -866,6 +866,12 @@ Status Doc::validate(const CollectionSchema::Ptr &schema,
"doc validate failed: field[", field_name,
"]'s sparse vector indices and values size not match");
}
if (sparse_indices.size() > kSparseMaxDimSize) {
return Status::InvalidArgument(
"doc validate failed: vector[", field_name,
"], the number of sparse indices exceeds the maximum limit ",
kSparseMaxDimSize);
}
}
break;
}
@ -881,6 +887,12 @@ Status Doc::validate(const CollectionSchema::Ptr &schema,
"doc validate failed: field[", field_name,
"]'s sparse vector indices and values size not match");
}
if (sparse_indices.size() > kSparseMaxDimSize) {
return Status::InvalidArgument(
"doc validate failed: vector[", field_name,
"], the number of sparse indices exceeds the maximum limit ",
kSparseMaxDimSize);
}
}
break;
}
@ -1251,9 +1263,11 @@ Status VectorQuery::validate(const FieldSchema *schema) const {
}
} else if (schema->is_sparse_vector()) {
// validate sparse indices size
if (query_sparse_indices_.size() >= kSparseMaxDimSize * sizeof(uint32_t)) {
if (query_sparse_indices_.size() > kSparseMaxDimSize * sizeof(uint32_t)) {
return Status::InvalidArgument(
"query validate failed: sparse indices size is too large");
"query validate failed: the number of sparse indices exceeds the "
"maximum limit ",
kSparseMaxDimSize);
}
} else {
return Status::InvalidArgument(

2
tests/db/index/common/doc_test.cc
View File

@ -1255,7 +1255,7 @@ TEST(VectorQuery, Validate) {
VectorQuery query;
query.field_name_ = "field_name";
query.topk_ = 100;
std::vector<uint32_t> query_indices = std::vector<uint32_t>(4097);
std::vector<uint32_t> query_indices = std::vector<uint32_t>(16385);
std::string query_indices_str =
std::string(reinterpret_cast<char *>(query_indices.data()),
query_indices.size() * sizeof(uint32_t));