fix: validate reformer/entity ex_bits consistency in HnswRabitqStreamer (#436)
When a RabitqReformer trained with one total_bits is reused for an entity configured with a different total_bits, the quantized data layout (ex_code size) silently mismatches. This causes get_full_est() to interpret binary quantization codes as floating-point factors, producing garbage distances that sporadically drop search results. Add an ex_bits() accessor to RabitqReformer and validate it against the entity's ex_bits in HnswRabitqStreamer::open(), returning IndexError_Mismatch on inconsistency. Fix the HNSWRabitqGeneral test to re-create a converter and reformer with matching total_bits=2 for the third invocation. Add TestExBitsMismatch to verify the mismatch is correctly rejected.
This commit is contained in:
@@ -318,6 +318,16 @@ int HnswRabitqStreamer::open(IndexStorage::Pointer stg) {
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Verify ex_bits consistency to avoid quantized data layout mismatch
|
||||
if (reformer_->ex_bits() != entity_.ex_bits()) {
|
||||
LOG_ERROR(
|
||||
"ex_bits mismatch between reformer(%zu) and entity(%zu). "
|
||||
"Reformer and entity must use the same total_bits configuration",
|
||||
reformer_->ex_bits(), (size_t)entity_.ex_bits());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
IndexMeta index_meta;
|
||||
ret = entity_.get_index_meta(&index_meta);
|
||||
if (ret == IndexError_NoExist) {
|
||||
|
||||
@@ -85,6 +85,10 @@ size_t RabitqReformer::num_clusters() const {
|
||||
return impl_->num_clusters;
|
||||
}
|
||||
|
||||
size_t RabitqReformer::ex_bits() const {
|
||||
return impl_->ex_bits;
|
||||
}
|
||||
|
||||
RabitqMetricType RabitqReformer::rabitq_metric_type() const {
|
||||
return Impl::from_rabitq(impl_->metric_type);
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ class RabitqReformer : public IndexReformer {
|
||||
HnswRabitqQueryEntity *entity) const;
|
||||
|
||||
size_t num_clusters() const;
|
||||
size_t ex_bits() const;
|
||||
RabitqMetricType rabitq_metric_type() const;
|
||||
|
||||
private:
|
||||
|
||||
Reference in New Issue
Block a user