feat: pass prefetch config (PO and PL) as search params (#482)

This commit is contained in:
luoxiaojian
2026-06-12 16:33:41 +08:00
committed by GitHub
parent 7c4571b8b9
commit 7a5f84e648
25 changed files with 957 additions and 54 deletions
+17 -12
View File
@@ -191,7 +191,8 @@ template <typename EntityType, typename HeapType>
void fast_search_neighbors(const EntityType &entity, HeapType &pool,
VisitFilter &visit, HnswDistCalculator &dc,
uint32_t topk, uint32_t ef, node_id_t entry_point,
dist_t entry_dist, uint32_t prefetch_lines) {
dist_t entry_dist, uint32_t prefetch_lines,
uint32_t prefetch_offset) {
const uint32_t max_deg = entity.max_degree(0); // level 0 only
const uint32_t cap = std::max(topk, ef);
pool.reset(static_cast<int32_t>(cap), static_cast<int32_t>(max_deg));
@@ -200,8 +201,6 @@ void fast_search_neighbors(const EntityType &entity, HeapType &pool,
visit.set_visited(entry_point);
pool.push_block(&entry_dist, &entry_point, 1);
static constexpr uint32_t GRAPH_PO = 8;
uint32_t buf_capacity = max_deg;
std::vector<node_id_t> neighbor_ids(buf_capacity);
std::vector<float> dists(buf_capacity);
@@ -221,7 +220,7 @@ void fast_search_neighbors(const EntityType &entity, HeapType &pool,
}
const uint32_t po =
std::min(static_cast<uint32_t>(neighbors.size()), GRAPH_PO);
std::min(static_cast<uint32_t>(neighbors.size()), prefetch_offset);
uint32_t unvisited_count = 0;
uint32_t i = 0;
@@ -270,8 +269,9 @@ void dual_heap_search_neighbors(const EntityType &entity, level_t level,
node_id_t *entry_point, dist_t *dist,
TopkHeap &topk, HnswContext *ctx,
HnswDistCalculator &dc, FilterFn &&filter) {
static constexpr uint32_t BATCH_SIZE = 12;
static constexpr uint32_t PREFETCH_STEP = 2;
const uint32_t prefetch_offset = ctx->po();
const uint32_t prefetch_lines =
ctx->pl() > 0 ? ctx->pl() : (entity.vector_size() + 63) / 64;
uint32_t buf_capacity = entity.max_degree(level);
std::vector<node_id_t> neighbor_ids(buf_capacity);
@@ -342,8 +342,12 @@ void dual_heap_search_neighbors(const EntityType &entity, level_t level,
}
// do prefetch
for (uint32_t i = 0; i < std::min(BATCH_SIZE * PREFETCH_STEP, size); ++i) {
ailego_prefetch(neighbor_vec_blocks[i].data());
for (uint32_t i = 0; i < std::min(prefetch_offset, size); ++i) {
const char *base =
static_cast<const char *>(neighbor_vec_blocks[i].data());
for (uint32_t cl = 0; cl < prefetch_lines; ++cl) {
ailego_prefetch(base + cl * 64);
}
}
for (uint32_t i = 0; i < size; ++i) {
@@ -388,8 +392,6 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
HnswDistCalculator &dc = ctx->dist_calculator();
const uint32_t prefetch_lines = (entity.vector_size() + 63) / 64;
if (!use_pool || ctx->filter().is_valid() || level != 0) {
// Dual-heap path: add_node, filtered search, or upper-level scan.
auto run_with_filter = [&](auto &&filter) {
@@ -410,6 +412,9 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
} else {
// Pool-based path for level-0 unfiltered search.
if constexpr (std::is_same_v<MemBlockType, MmapMemoryBlock>) {
const uint32_t prefetch_lines =
ctx->pl() > 0 ? ctx->pl() : (entity.vector_size() + 63) / 64;
// Fast path: direct pointer access via get_vector_ptr.
// BlockHeap (AVX2) or LinearPool (scalar) for top-k tracking.
const uint32_t topk_v = static_cast<uint32_t>(ctx->topk());
@@ -422,12 +427,12 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
if (avx2_ok) {
auto &bpool = ctx->block_pool();
fast_search_neighbors(entity, bpool, visit, dc, topk_v, ef_v,
*entry_point, *dist, prefetch_lines);
*entry_point, *dist, prefetch_lines, ctx->po());
copy_pool_to_topk(bpool, topk);
} else {
auto &lpool = ctx->pool();
fast_search_neighbors(entity, lpool, visit, dc, topk_v, ef_v,
*entry_point, *dist, prefetch_lines);
*entry_point, *dist, prefetch_lines, ctx->po());
copy_pool_to_topk(lpool, topk);
}
} else {
+10
View File
@@ -151,6 +151,14 @@ int HnswContext::update(const ailego::Params &params) {
topk_heap_.limit(std::max(topk_, ef_));
}
if (params.has(PARAM_HNSW_SEARCHER_PO)) {
params.get(PARAM_HNSW_SEARCHER_PO, &po_);
}
if (params.has(PARAM_HNSW_SEARCHER_PL)) {
params.get(PARAM_HNSW_SEARCHER_PL, &pl_);
}
if (params.has(PARAM_HNSW_SEARCHER_MAX_SCAN_RATIO)) {
params.get(PARAM_HNSW_SEARCHER_MAX_SCAN_RATIO, &max_scan_ratio_);
max_scan_num_ =
@@ -171,6 +179,8 @@ int HnswContext::update(const ailego::Params &params) {
topk_heap_.limit(std::max(topk_, ef_));
}
params.get(PARAM_HNSW_STREAMER_EF, &ef_);
params.get(PARAM_HNSW_STREAMER_PO, &po_);
params.get(PARAM_HNSW_STREAMER_PL, &pl_);
params.get(PARAM_HNSW_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
params.get(PARAM_HNSW_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
params.get(PARAM_HNSW_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
+18
View File
@@ -314,6 +314,22 @@ class HnswContext : public IndexContext {
return ef_;
}
inline void set_po(uint32_t v) {
po_ = v;
}
inline uint32_t po(void) const {
return po_;
}
inline void set_pl(uint32_t v) {
pl_ = v;
}
inline uint32_t pl(void) const {
return pl_;
}
inline void set_filter_mode(uint32_t v) {
filter_mode_ = v;
}
@@ -524,6 +540,8 @@ class HnswContext : public IndexContext {
uint32_t filter_mode_{VisitFilter::ByteMap};
float negative_probability_{HnswEntity::kDefaultBFNegativeProbability};
uint32_t ef_{HnswEntity::kDefaultEf};
uint32_t po_{8};
uint32_t pl_{0};
float max_scan_ratio_{HnswEntity::kDefaultScanRatio};
uint32_t magic_{0U};
std::vector<IndexDocumentList> results_{};
+4
View File
@@ -38,6 +38,8 @@ static const std::string PARAM_HNSW_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
"proxima.hnsw.builder.l0_max_neighbor_count_multiplier");
static const std::string PARAM_HNSW_SEARCHER_EF("proxima.hnsw.searcher.ef");
static const std::string PARAM_HNSW_SEARCHER_PO("proxima.hnsw.searcher.po");
static const std::string PARAM_HNSW_SEARCHER_PL("proxima.hnsw.searcher.pl");
static const std::string PARAM_HNSW_SEARCHER_BRUTE_FORCE_THRESHOLD(
"proxima.hnsw.searcher.brute_force_threshold");
static const std::string PARAM_HNSW_SEARCHER_NEIGHBORS_IN_MEMORY_ENABLE(
@@ -60,6 +62,8 @@ static const std::string PARAM_HNSW_STREAMER_MIN_SCAN_LIMIT(
static const std::string PARAM_HNSW_STREAMER_MAX_SCAN_LIMIT(
"proxima.hnsw.streamer.max_scan_limit");
static const std::string PARAM_HNSW_STREAMER_EF("proxima.hnsw.streamer.ef");
static const std::string PARAM_HNSW_STREAMER_PO("proxima.hnsw.streamer.po");
static const std::string PARAM_HNSW_STREAMER_PL("proxima.hnsw.streamer.pl");
static const std::string PARAM_HNSW_STREAMER_EFCONSTRUCTION(
"proxima.hnsw.streamer.efconstruction");
static const std::string PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT(
+4
View File
@@ -52,6 +52,8 @@ int HnswStreamer::init(const IndexMeta &imeta, const ailego::Params &params) {
params.get(PARAM_HNSW_STREAMER_DOCS_HARD_LIMIT, &docs_hard_limit_);
params.get(PARAM_HNSW_STREAMER_EF, &ef_);
params.get(PARAM_HNSW_STREAMER_PO, &po_);
params.get(PARAM_HNSW_STREAMER_PL, &pl_);
params.get(PARAM_HNSW_STREAMER_EFCONSTRUCTION, &ef_construction_);
params.get(PARAM_HNSW_STREAMER_VISIT_BLOOMFILTER_ENABLE, &bf_enabled_);
params.get(PARAM_HNSW_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB,
@@ -417,6 +419,8 @@ IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
return Context::Pointer();
}
ctx->set_ef(ef_);
ctx->set_po(po_);
ctx->set_pl(pl_);
ctx->set_max_scan_limit(max_scan_limit_);
ctx->set_min_scan_limit(min_scan_limit_);
ctx->set_max_scan_ratio(max_scan_ratio_);
+2
View File
@@ -215,6 +215,8 @@ class HnswStreamer : public IndexStreamer {
uint32_t upper_max_neighbor_cnt_{HnswEntity::kDefaultUpperMaxNeighborCnt};
uint32_t l0_max_neighbor_cnt_{HnswEntity::kDefaultL0MaxNeighborCnt};
uint32_t ef_{HnswEntity::kDefaultEf};
uint32_t po_{8};
uint32_t pl_{0};
uint32_t ef_construction_{HnswEntity::kDefaultEfConstruction};
uint32_t scaling_factor_{HnswEntity::kDefaultScalingFactor};
size_t bruteforce_threshold_{HnswEntity::kDefaultBruteForceThreshold};
+15 -22
View File
@@ -126,7 +126,7 @@ template <typename EntityType, typename HeapType>
void fast_greedy_search(const EntityType &entity, HeapType &pool,
VisitFilter &visit, VamanaDistCalculator &dc,
uint32_t topk, uint32_t ef, node_id_t entry_point,
uint32_t prefetch_lines) {
uint32_t prefetch_lines, uint32_t prefetch_offset) {
const uint32_t max_deg = entity.max_degree();
const uint32_t cap = std::max(topk, ef);
pool.reset(static_cast<int32_t>(cap), static_cast<int32_t>(max_deg));
@@ -136,8 +136,6 @@ void fast_greedy_search(const EntityType &entity, HeapType &pool,
visit.set_visited(entry_point);
pool.push_block(&ep_dist, &entry_point, 1);
static constexpr uint32_t GRAPH_PO = 8;
uint32_t buf_capacity = max_deg;
std::vector<node_id_t> neighbor_ids(buf_capacity);
std::vector<float> dists(buf_capacity);
@@ -157,7 +155,7 @@ void fast_greedy_search(const EntityType &entity, HeapType &pool,
}
const uint32_t po =
std::min(static_cast<uint32_t>(neighbors.size()), GRAPH_PO);
std::min(static_cast<uint32_t>(neighbors.size()), prefetch_offset);
uint32_t unvisited_count = 0;
uint32_t i = 0;
@@ -200,8 +198,9 @@ template <typename EntityType, typename MemBlockType, typename FilterFn>
void dual_heap_greedy_search(const EntityType &entity, VamanaContext *ctx,
VamanaDistCalculator &dc, node_id_t entry_point,
FilterFn &&filter) {
static constexpr uint32_t PREFETCH_BATCH = 2;
static constexpr uint32_t PREFETCH_STEP = 2;
const uint32_t prefetch_offset = ctx->po();
const uint32_t prefetch_lines =
ctx->pl() > 0 ? ctx->pl() : (entity.vector_size() + 63) / 64;
uint32_t buf_capacity = entity.max_degree();
std::vector<node_id_t> neighbor_ids(buf_capacity);
@@ -273,9 +272,12 @@ void dual_heap_greedy_search(const EntityType &entity, VamanaContext *ctx,
neighbor_vec_blocks);
if (ailego_unlikely(ret != 0)) break;
for (uint32_t i = 0;
i < std::min(PREFETCH_BATCH * PREFETCH_STEP, unvisited_count); ++i) {
ailego_prefetch(neighbor_vec_blocks[i].data());
for (uint32_t i = 0; i < std::min(prefetch_offset, unvisited_count); ++i) {
const char *base =
static_cast<const char *>(neighbor_vec_blocks[i].data());
for (uint32_t cl = 0; cl < prefetch_lines; ++cl) {
ailego_prefetch(base + cl * 64);
}
}
// Batch distance computation (reuse pre-allocated buffers).
@@ -316,17 +318,8 @@ void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
const IndexFilter &index_filter =
static_cast<const IndexContext *>(ctx)->filter();
// Number of cache lines per vector (e.g. 2 for dim=128).
// Used by both the fallback candidates/filter path and the fast helpers.
uint32_t prefetch_lines = (dc.dimension() + 63) / 64;
if constexpr (std::is_same_v<EntityType, VamanaContiguousStreamerEntity>) {
// Contiguous flat array stride is already 64B-aligned. Use it so that
// prefetch does not overshoot into the next vector.
size_t stride = entity.vector_stride();
if (stride > 0) {
prefetch_lines = static_cast<uint32_t>(stride / 64);
}
}
const uint32_t prefetch_lines =
ctx->pl() > 0 ? ctx->pl() : (entity.vector_size() + 63) / 64;
if (!use_pool || index_filter.is_valid()) {
// Fallback path used by add_node (use_pool=false) and filtered search.
@@ -362,12 +355,12 @@ void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
if (avx2_ok) {
auto &bpool = ctx->block_pool();
fast_greedy_search(entity, bpool, visit, dc, topk_v, ef_v, entry_point,
prefetch_lines);
prefetch_lines, ctx->po());
copy_pool_to_topk(bpool, topk_heap);
} else {
auto &lpool = ctx->pool();
fast_greedy_search(entity, lpool, visit, dc, topk_v, ef_v, entry_point,
prefetch_lines);
prefetch_lines, ctx->po());
copy_pool_to_topk(lpool, topk_heap);
}
} else {
@@ -119,6 +119,12 @@ int VamanaContext::update(const ailego::Params &params) {
params.get(PARAM_VAMANA_STREAMER_EF, &ef);
ef_ = ef;
topk_heap_.limit(std::max(topk_, ef_));
uint32_t po = po_;
params.get(PARAM_VAMANA_STREAMER_PO, &po);
po_ = po;
uint32_t pl = pl_;
params.get(PARAM_VAMANA_STREAMER_PL, &pl);
pl_ = pl;
return 0;
}
@@ -177,6 +177,21 @@ class VamanaContext : public IndexContext {
inline uint32_t ef() const {
return ef_;
}
inline void set_po(uint32_t v) {
po_ = v;
}
inline uint32_t po() const {
return po_;
}
inline void set_pl(uint32_t v) {
pl_ = v;
}
inline uint32_t pl() const {
return pl_;
}
inline void set_max_scan_ratio(float v) {
max_scan_ratio_ = v;
}
@@ -296,6 +311,8 @@ class VamanaContext : public IndexContext {
uint32_t reserve_max_doc_cnt_{kMinReserveDocCnt};
uint32_t topk_{0};
uint32_t ef_{VamanaEntity::kDefaultEf};
uint32_t po_{8};
uint32_t pl_{0};
float max_scan_ratio_{VamanaEntity::kDefaultScanRatio};
size_t max_scan_limit_{VamanaEntity::kDefaultMaxScanLimit};
size_t min_scan_limit_{VamanaEntity::kDefaultMinScanLimit};
@@ -48,6 +48,8 @@ static const std::string PARAM_VAMANA_STREAMER_ALPHA(
static const std::string PARAM_VAMANA_STREAMER_MAX_OCCLUSION_SIZE(
"proxima.vamana.streamer.max_occlusion_size");
static const std::string PARAM_VAMANA_STREAMER_EF("proxima.vamana.streamer.ef");
static const std::string PARAM_VAMANA_STREAMER_PO("proxima.vamana.streamer.po");
static const std::string PARAM_VAMANA_STREAMER_PL("proxima.vamana.streamer.pl");
static const std::string PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD(
"proxima.vamana.streamer.brute_force_threshold");
static const std::string PARAM_VAMANA_STREAMER_MAX_SCAN_RATIO(