fix: some bugs in ivf index (#524)
This commit is contained in:
@@ -70,6 +70,8 @@ static const std::string PARAM_IVF_SEARCHER_OPTIMIZER_PARAMS(
|
||||
"proxima.ivf.searcher.optimizer_params");
|
||||
static const std::string PARAM_IVF_SEARCHER_CONVERTER_REFORMER(
|
||||
"proxima.ivf.searcher.converter_reformer");
|
||||
static const std::string PARAM_IVF_SEARCHER_NPROBE(
|
||||
"proxima.ivf.searcher.nprobe");
|
||||
|
||||
// Constants
|
||||
static constexpr char const *kIPMetricName = "InnerProduct";
|
||||
|
||||
@@ -71,9 +71,30 @@ class IVFSearcherContext : public IndexSearcher::Context {
|
||||
std::max(static_cast<uint32_t>(
|
||||
std::round(entity_->inverted_list_count() * scan_ratio_)),
|
||||
1u);
|
||||
|
||||
uint32_t nprobe = 0;
|
||||
params.get(PARAM_IVF_SEARCHER_NPROBE, &nprobe);
|
||||
if (nprobe > 0) {
|
||||
nprobe = std::min(nprobe,
|
||||
static_cast<uint32_t>(entity_->inverted_list_count()));
|
||||
topk_val = nprobe;
|
||||
}
|
||||
|
||||
centroid_searcher_ctx_->set_topk(topk_val);
|
||||
max_scan_count_ =
|
||||
static_cast<uint32_t>(std::ceil(entity_->vector_count() * scan_ratio_));
|
||||
|
||||
// When nprobe is explicitly set, scale max_scan_count proportionally
|
||||
// to ensure all probed clusters can be fully scanned.
|
||||
if (nprobe > 0 && entity_->inverted_list_count() > 0) {
|
||||
uint32_t list_count =
|
||||
static_cast<uint32_t>(entity_->inverted_list_count());
|
||||
max_scan_count_ = static_cast<uint32_t>(
|
||||
(static_cast<uint64_t>(entity_->vector_count()) * nprobe +
|
||||
list_count - 1) /
|
||||
list_count);
|
||||
} else {
|
||||
max_scan_count_ = static_cast<uint32_t>(
|
||||
std::ceil(entity_->vector_count() * scan_ratio_));
|
||||
}
|
||||
max_scan_count_ = std::max(bruteforce_threshold_, max_scan_count_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -211,9 +211,7 @@ int IVFIndex::_prepare_for_search(
|
||||
if (ivf_search_param->nprobe > 0) {
|
||||
// TODO: 1. sparse; 2. default ef
|
||||
ailego::Params params;
|
||||
// need fix
|
||||
params.set(core::PARAM_IVF_BUILDER_CENTROID_COUNT,
|
||||
ivf_search_param->nprobe);
|
||||
params.set(core::PARAM_IVF_SEARCHER_NPROBE, ivf_search_param->nprobe);
|
||||
context->update(params);
|
||||
}
|
||||
return 0;
|
||||
|
||||
Reference in New Issue
Block a user