feat: enlarge indice size limit for sparse vectors (#229)
This commit is contained in:
@@ -195,8 +195,10 @@ int FlatSparseStreamer::add_impl(uint64_t pkey, const uint32_t sparse_count,
|
||||
}
|
||||
|
||||
if (ailego_unlikely(sparse_count > PARAM_FLAT_SPARSE_MAX_DIM_SIZE)) {
|
||||
LOG_ERROR("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
|
||||
sparse_count, (size_t)pkey);
|
||||
LOG_ERROR(
|
||||
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
|
||||
"maximum allowed (%u), key=%zu",
|
||||
sparse_count, PARAM_FLAT_SPARSE_MAX_DIM_SIZE, (size_t)pkey);
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_InvalidValue;
|
||||
}
|
||||
@@ -252,8 +254,10 @@ int FlatSparseStreamer::add_with_id_impl(uint32_t pkey,
|
||||
}
|
||||
|
||||
if (ailego_unlikely(sparse_count > PARAM_FLAT_SPARSE_MAX_DIM_SIZE)) {
|
||||
LOG_ERROR("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
|
||||
sparse_count, (size_t)pkey);
|
||||
LOG_ERROR(
|
||||
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
|
||||
"maximum allowed (%u), key=%zu",
|
||||
sparse_count, PARAM_FLAT_SPARSE_MAX_DIM_SIZE, (size_t)pkey);
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_InvalidValue;
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
static constexpr uint32_t PARAM_FLAT_SPARSE_MAX_DIM_SIZE = 4096;
|
||||
static constexpr uint32_t PARAM_FLAT_SPARSE_MAX_DIM_SIZE = 16384;
|
||||
static const std::string PARAM_FLAT_SPARSE_META_SEG_ID =
|
||||
"bruteforce_sparse_meta";
|
||||
|
||||
|
||||
@@ -88,9 +88,11 @@ int HnswSparseBuilderEntity::add_vector(level_t level, key_t key,
|
||||
const uint32_t sparse_count,
|
||||
const uint32_t *sparse_indices,
|
||||
const void *sparse_vec, node_id_t *id) {
|
||||
if (ailego_unlikely(sparse_count >= HnswSparseEntity::kSparseMaxDimSize)) {
|
||||
LOG_WARN("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
|
||||
sparse_count, (size_t)key);
|
||||
if (ailego_unlikely(sparse_count > HnswSparseEntity::kSparseMaxDimSize)) {
|
||||
LOG_WARN(
|
||||
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
|
||||
"maximum allowed (%u), key=%zu",
|
||||
sparse_count, HnswSparseEntity::kSparseMaxDimSize, (size_t)key);
|
||||
return IndexError_InvalidValue;
|
||||
}
|
||||
|
||||
|
||||
@@ -610,7 +610,7 @@ class HnswSparseEntity {
|
||||
|
||||
constexpr static uint32_t kSparseMetaSize = 2u * sizeof(uint64_t);
|
||||
constexpr static float kDefaultSparseNeighborRatio = 0.5f;
|
||||
constexpr static uint32_t kSparseMaxDimSize = 4096;
|
||||
constexpr static uint32_t kSparseMaxDimSize = 16384;
|
||||
constexpr static float kDefaultQueryFilteringRatio = 0.0f; // turn off
|
||||
|
||||
protected:
|
||||
|
||||
@@ -438,9 +438,11 @@ int HnswSparseStreamer::add_with_id_impl(uint32_t id,
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(sparse_count >= HnswSparseEntity::kSparseMaxDimSize)) {
|
||||
LOG_WARN("Add vector failed, dim size too larg, dim_size=%u, id=%u",
|
||||
sparse_count, id);
|
||||
if (ailego_unlikely(sparse_count > HnswSparseEntity::kSparseMaxDimSize)) {
|
||||
LOG_WARN(
|
||||
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
|
||||
"maximum allowed (%u), id=%u",
|
||||
sparse_count, HnswSparseEntity::kSparseMaxDimSize, id);
|
||||
return IndexError_InvalidValue;
|
||||
}
|
||||
|
||||
@@ -523,9 +525,11 @@ int HnswSparseStreamer::add_impl(uint64_t pkey, const uint32_t sparse_count,
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (ailego_unlikely(sparse_count >= HnswSparseEntity::kSparseMaxDimSize)) {
|
||||
LOG_WARN("Add vector failed, dim size too larg, dim_size=%u, key=%zu",
|
||||
sparse_count, (size_t)pkey);
|
||||
if (ailego_unlikely(sparse_count > HnswSparseEntity::kSparseMaxDimSize)) {
|
||||
LOG_WARN(
|
||||
"Failed to add sparse vector: number of non-zero elements (%u) exceeds "
|
||||
"maximum allowed (%u), key=%zu",
|
||||
sparse_count, HnswSparseEntity::kSparseMaxDimSize, (size_t)pkey);
|
||||
return IndexError_InvalidValue;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user