feat: enlarge indice size limit for sparse vectors (#229)

This commit is contained in:
Qinren Zhou
2026-03-16 17:12:24 +08:00
committed by GitHub
parent 5124fbc072
commit 3c1241f7c6
8 changed files with 43 additions and 19 deletions
@@ -195,8 +195,10 @@ int FlatSparseStreamer::add_impl(uint64_t pkey, const uint32_t sparse_count,
}
if (ailego_unlikely(sparse_count > PARAM_FLAT_SPARSE_MAX_DIM_SIZE)) {
LOG_ERROR("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
sparse_count, (size_t)pkey);
LOG_ERROR(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), key=%zu",
sparse_count, PARAM_FLAT_SPARSE_MAX_DIM_SIZE, (size_t)pkey);
(*stats_.mutable_discarded_count())++;
return IndexError_InvalidValue;
}
@@ -252,8 +254,10 @@ int FlatSparseStreamer::add_with_id_impl(uint32_t pkey,
}
if (ailego_unlikely(sparse_count > PARAM_FLAT_SPARSE_MAX_DIM_SIZE)) {
LOG_ERROR("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
sparse_count, (size_t)pkey);
LOG_ERROR(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), key=%zu",
sparse_count, PARAM_FLAT_SPARSE_MAX_DIM_SIZE, (size_t)pkey);
(*stats_.mutable_discarded_count())++;
return IndexError_InvalidValue;
}
@@ -19,7 +19,7 @@
namespace zvec {
namespace core {
static constexpr uint32_t PARAM_FLAT_SPARSE_MAX_DIM_SIZE = 4096;
static constexpr uint32_t PARAM_FLAT_SPARSE_MAX_DIM_SIZE = 16384;
static const std::string PARAM_FLAT_SPARSE_META_SEG_ID =
"bruteforce_sparse_meta";
@@ -88,9 +88,11 @@ int HnswSparseBuilderEntity::add_vector(level_t level, key_t key,
const uint32_t sparse_count,
const uint32_t *sparse_indices,
const void *sparse_vec, node_id_t *id) {
if (ailego_unlikely(sparse_count >= HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
sparse_count, (size_t)key);
if (ailego_unlikely(sparse_count > HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), key=%zu",
sparse_count, HnswSparseEntity::kSparseMaxDimSize, (size_t)key);
return IndexError_InvalidValue;
}
@@ -610,7 +610,7 @@ class HnswSparseEntity {
constexpr static uint32_t kSparseMetaSize = 2u * sizeof(uint64_t);
constexpr static float kDefaultSparseNeighborRatio = 0.5f;
constexpr static uint32_t kSparseMaxDimSize = 4096;
constexpr static uint32_t kSparseMaxDimSize = 16384;
constexpr static float kDefaultQueryFilteringRatio = 0.0f; // turn off
protected:
@@ -438,9 +438,11 @@ int HnswSparseStreamer::add_with_id_impl(uint32_t id,
return ret;
}
if (ailego_unlikely(sparse_count >= HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN("Add vector failed, dim size too larg, dim_size=%u, id=%u",
sparse_count, id);
if (ailego_unlikely(sparse_count > HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), id=%u",
sparse_count, HnswSparseEntity::kSparseMaxDimSize, id);
return IndexError_InvalidValue;
}
@@ -523,9 +525,11 @@ int HnswSparseStreamer::add_impl(uint64_t pkey, const uint32_t sparse_count,
return ret;
}
if (ailego_unlikely(sparse_count >= HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
sparse_count, (size_t)pkey);
if (ailego_unlikely(sparse_count > HnswSparseEntity::kSparseMaxDimSize)) {
LOG_WARN(
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
"maximum allowed (%u), key=%zu",
sparse_count, HnswSparseEntity::kSparseMaxDimSize, (size_t)pkey);
return IndexError_InvalidValue;
}