feat: pass prefetch config (PO and PL) as search params (#482)
This commit is contained in:
@@ -191,7 +191,8 @@ template <typename EntityType, typename HeapType>
|
||||
void fast_search_neighbors(const EntityType &entity, HeapType &pool,
|
||||
VisitFilter &visit, HnswDistCalculator &dc,
|
||||
uint32_t topk, uint32_t ef, node_id_t entry_point,
|
||||
dist_t entry_dist, uint32_t prefetch_lines) {
|
||||
dist_t entry_dist, uint32_t prefetch_lines,
|
||||
uint32_t prefetch_offset) {
|
||||
const uint32_t max_deg = entity.max_degree(0); // level 0 only
|
||||
const uint32_t cap = std::max(topk, ef);
|
||||
pool.reset(static_cast<int32_t>(cap), static_cast<int32_t>(max_deg));
|
||||
@@ -200,8 +201,6 @@ void fast_search_neighbors(const EntityType &entity, HeapType &pool,
|
||||
visit.set_visited(entry_point);
|
||||
pool.push_block(&entry_dist, &entry_point, 1);
|
||||
|
||||
static constexpr uint32_t GRAPH_PO = 8;
|
||||
|
||||
uint32_t buf_capacity = max_deg;
|
||||
std::vector<node_id_t> neighbor_ids(buf_capacity);
|
||||
std::vector<float> dists(buf_capacity);
|
||||
@@ -221,7 +220,7 @@ void fast_search_neighbors(const EntityType &entity, HeapType &pool,
|
||||
}
|
||||
|
||||
const uint32_t po =
|
||||
std::min(static_cast<uint32_t>(neighbors.size()), GRAPH_PO);
|
||||
std::min(static_cast<uint32_t>(neighbors.size()), prefetch_offset);
|
||||
uint32_t unvisited_count = 0;
|
||||
uint32_t i = 0;
|
||||
|
||||
@@ -270,8 +269,9 @@ void dual_heap_search_neighbors(const EntityType &entity, level_t level,
|
||||
node_id_t *entry_point, dist_t *dist,
|
||||
TopkHeap &topk, HnswContext *ctx,
|
||||
HnswDistCalculator &dc, FilterFn &&filter) {
|
||||
static constexpr uint32_t BATCH_SIZE = 12;
|
||||
static constexpr uint32_t PREFETCH_STEP = 2;
|
||||
const uint32_t prefetch_offset = ctx->po();
|
||||
const uint32_t prefetch_lines =
|
||||
ctx->pl() > 0 ? ctx->pl() : (entity.vector_size() + 63) / 64;
|
||||
|
||||
uint32_t buf_capacity = entity.max_degree(level);
|
||||
std::vector<node_id_t> neighbor_ids(buf_capacity);
|
||||
@@ -342,8 +342,12 @@ void dual_heap_search_neighbors(const EntityType &entity, level_t level,
|
||||
}
|
||||
|
||||
// do prefetch
|
||||
for (uint32_t i = 0; i < std::min(BATCH_SIZE * PREFETCH_STEP, size); ++i) {
|
||||
ailego_prefetch(neighbor_vec_blocks[i].data());
|
||||
for (uint32_t i = 0; i < std::min(prefetch_offset, size); ++i) {
|
||||
const char *base =
|
||||
static_cast<const char *>(neighbor_vec_blocks[i].data());
|
||||
for (uint32_t cl = 0; cl < prefetch_lines; ++cl) {
|
||||
ailego_prefetch(base + cl * 64);
|
||||
}
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
@@ -388,8 +392,6 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
|
||||
const auto &entity = static_cast<const EntityType &>(ctx->get_entity());
|
||||
HnswDistCalculator &dc = ctx->dist_calculator();
|
||||
|
||||
const uint32_t prefetch_lines = (entity.vector_size() + 63) / 64;
|
||||
|
||||
if (!use_pool || ctx->filter().is_valid() || level != 0) {
|
||||
// Dual-heap path: add_node, filtered search, or upper-level scan.
|
||||
auto run_with_filter = [&](auto &&filter) {
|
||||
@@ -410,6 +412,9 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
|
||||
} else {
|
||||
// Pool-based path for level-0 unfiltered search.
|
||||
if constexpr (std::is_same_v<MemBlockType, MmapMemoryBlock>) {
|
||||
const uint32_t prefetch_lines =
|
||||
ctx->pl() > 0 ? ctx->pl() : (entity.vector_size() + 63) / 64;
|
||||
|
||||
// Fast path: direct pointer access via get_vector_ptr.
|
||||
// BlockHeap (AVX2) or LinearPool (scalar) for top-k tracking.
|
||||
const uint32_t topk_v = static_cast<uint32_t>(ctx->topk());
|
||||
@@ -422,12 +427,12 @@ void HnswAlgorithm<EntityType>::search_neighbors(level_t level,
|
||||
if (avx2_ok) {
|
||||
auto &bpool = ctx->block_pool();
|
||||
fast_search_neighbors(entity, bpool, visit, dc, topk_v, ef_v,
|
||||
*entry_point, *dist, prefetch_lines);
|
||||
*entry_point, *dist, prefetch_lines, ctx->po());
|
||||
copy_pool_to_topk(bpool, topk);
|
||||
} else {
|
||||
auto &lpool = ctx->pool();
|
||||
fast_search_neighbors(entity, lpool, visit, dc, topk_v, ef_v,
|
||||
*entry_point, *dist, prefetch_lines);
|
||||
*entry_point, *dist, prefetch_lines, ctx->po());
|
||||
copy_pool_to_topk(lpool, topk);
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -151,6 +151,14 @@ int HnswContext::update(const ailego::Params ¶ms) {
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
}
|
||||
|
||||
if (params.has(PARAM_HNSW_SEARCHER_PO)) {
|
||||
params.get(PARAM_HNSW_SEARCHER_PO, &po_);
|
||||
}
|
||||
|
||||
if (params.has(PARAM_HNSW_SEARCHER_PL)) {
|
||||
params.get(PARAM_HNSW_SEARCHER_PL, &pl_);
|
||||
}
|
||||
|
||||
if (params.has(PARAM_HNSW_SEARCHER_MAX_SCAN_RATIO)) {
|
||||
params.get(PARAM_HNSW_SEARCHER_MAX_SCAN_RATIO, &max_scan_ratio_);
|
||||
max_scan_num_ =
|
||||
@@ -171,6 +179,8 @@ int HnswContext::update(const ailego::Params ¶ms) {
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
}
|
||||
params.get(PARAM_HNSW_STREAMER_EF, &ef_);
|
||||
params.get(PARAM_HNSW_STREAMER_PO, &po_);
|
||||
params.get(PARAM_HNSW_STREAMER_PL, &pl_);
|
||||
params.get(PARAM_HNSW_STREAMER_MAX_SCAN_RATIO, &max_scan_ratio_);
|
||||
params.get(PARAM_HNSW_STREAMER_MAX_SCAN_LIMIT, &max_scan_limit_);
|
||||
params.get(PARAM_HNSW_STREAMER_MIN_SCAN_LIMIT, &min_scan_limit_);
|
||||
|
||||
@@ -314,6 +314,22 @@ class HnswContext : public IndexContext {
|
||||
return ef_;
|
||||
}
|
||||
|
||||
inline void set_po(uint32_t v) {
|
||||
po_ = v;
|
||||
}
|
||||
|
||||
inline uint32_t po(void) const {
|
||||
return po_;
|
||||
}
|
||||
|
||||
inline void set_pl(uint32_t v) {
|
||||
pl_ = v;
|
||||
}
|
||||
|
||||
inline uint32_t pl(void) const {
|
||||
return pl_;
|
||||
}
|
||||
|
||||
inline void set_filter_mode(uint32_t v) {
|
||||
filter_mode_ = v;
|
||||
}
|
||||
@@ -524,6 +540,8 @@ class HnswContext : public IndexContext {
|
||||
uint32_t filter_mode_{VisitFilter::ByteMap};
|
||||
float negative_probability_{HnswEntity::kDefaultBFNegativeProbability};
|
||||
uint32_t ef_{HnswEntity::kDefaultEf};
|
||||
uint32_t po_{8};
|
||||
uint32_t pl_{0};
|
||||
float max_scan_ratio_{HnswEntity::kDefaultScanRatio};
|
||||
uint32_t magic_{0U};
|
||||
std::vector<IndexDocumentList> results_{};
|
||||
|
||||
@@ -38,6 +38,8 @@ static const std::string PARAM_HNSW_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
|
||||
"proxima.hnsw.builder.l0_max_neighbor_count_multiplier");
|
||||
|
||||
static const std::string PARAM_HNSW_SEARCHER_EF("proxima.hnsw.searcher.ef");
|
||||
static const std::string PARAM_HNSW_SEARCHER_PO("proxima.hnsw.searcher.po");
|
||||
static const std::string PARAM_HNSW_SEARCHER_PL("proxima.hnsw.searcher.pl");
|
||||
static const std::string PARAM_HNSW_SEARCHER_BRUTE_FORCE_THRESHOLD(
|
||||
"proxima.hnsw.searcher.brute_force_threshold");
|
||||
static const std::string PARAM_HNSW_SEARCHER_NEIGHBORS_IN_MEMORY_ENABLE(
|
||||
@@ -60,6 +62,8 @@ static const std::string PARAM_HNSW_STREAMER_MIN_SCAN_LIMIT(
|
||||
static const std::string PARAM_HNSW_STREAMER_MAX_SCAN_LIMIT(
|
||||
"proxima.hnsw.streamer.max_scan_limit");
|
||||
static const std::string PARAM_HNSW_STREAMER_EF("proxima.hnsw.streamer.ef");
|
||||
static const std::string PARAM_HNSW_STREAMER_PO("proxima.hnsw.streamer.po");
|
||||
static const std::string PARAM_HNSW_STREAMER_PL("proxima.hnsw.streamer.pl");
|
||||
static const std::string PARAM_HNSW_STREAMER_EFCONSTRUCTION(
|
||||
"proxima.hnsw.streamer.efconstruction");
|
||||
static const std::string PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT(
|
||||
|
||||
@@ -52,6 +52,8 @@ int HnswStreamer::init(const IndexMeta &imeta, const ailego::Params ¶ms) {
|
||||
|
||||
params.get(PARAM_HNSW_STREAMER_DOCS_HARD_LIMIT, &docs_hard_limit_);
|
||||
params.get(PARAM_HNSW_STREAMER_EF, &ef_);
|
||||
params.get(PARAM_HNSW_STREAMER_PO, &po_);
|
||||
params.get(PARAM_HNSW_STREAMER_PL, &pl_);
|
||||
params.get(PARAM_HNSW_STREAMER_EFCONSTRUCTION, &ef_construction_);
|
||||
params.get(PARAM_HNSW_STREAMER_VISIT_BLOOMFILTER_ENABLE, &bf_enabled_);
|
||||
params.get(PARAM_HNSW_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB,
|
||||
@@ -417,6 +419,8 @@ IndexStreamer::Context::Pointer HnswStreamer::create_context(void) const {
|
||||
return Context::Pointer();
|
||||
}
|
||||
ctx->set_ef(ef_);
|
||||
ctx->set_po(po_);
|
||||
ctx->set_pl(pl_);
|
||||
ctx->set_max_scan_limit(max_scan_limit_);
|
||||
ctx->set_min_scan_limit(min_scan_limit_);
|
||||
ctx->set_max_scan_ratio(max_scan_ratio_);
|
||||
|
||||
@@ -215,6 +215,8 @@ class HnswStreamer : public IndexStreamer {
|
||||
uint32_t upper_max_neighbor_cnt_{HnswEntity::kDefaultUpperMaxNeighborCnt};
|
||||
uint32_t l0_max_neighbor_cnt_{HnswEntity::kDefaultL0MaxNeighborCnt};
|
||||
uint32_t ef_{HnswEntity::kDefaultEf};
|
||||
uint32_t po_{8};
|
||||
uint32_t pl_{0};
|
||||
uint32_t ef_construction_{HnswEntity::kDefaultEfConstruction};
|
||||
uint32_t scaling_factor_{HnswEntity::kDefaultScalingFactor};
|
||||
size_t bruteforce_threshold_{HnswEntity::kDefaultBruteForceThreshold};
|
||||
|
||||
@@ -126,7 +126,7 @@ template <typename EntityType, typename HeapType>
|
||||
void fast_greedy_search(const EntityType &entity, HeapType &pool,
|
||||
VisitFilter &visit, VamanaDistCalculator &dc,
|
||||
uint32_t topk, uint32_t ef, node_id_t entry_point,
|
||||
uint32_t prefetch_lines) {
|
||||
uint32_t prefetch_lines, uint32_t prefetch_offset) {
|
||||
const uint32_t max_deg = entity.max_degree();
|
||||
const uint32_t cap = std::max(topk, ef);
|
||||
pool.reset(static_cast<int32_t>(cap), static_cast<int32_t>(max_deg));
|
||||
@@ -136,8 +136,6 @@ void fast_greedy_search(const EntityType &entity, HeapType &pool,
|
||||
visit.set_visited(entry_point);
|
||||
pool.push_block(&ep_dist, &entry_point, 1);
|
||||
|
||||
static constexpr uint32_t GRAPH_PO = 8;
|
||||
|
||||
uint32_t buf_capacity = max_deg;
|
||||
std::vector<node_id_t> neighbor_ids(buf_capacity);
|
||||
std::vector<float> dists(buf_capacity);
|
||||
@@ -157,7 +155,7 @@ void fast_greedy_search(const EntityType &entity, HeapType &pool,
|
||||
}
|
||||
|
||||
const uint32_t po =
|
||||
std::min(static_cast<uint32_t>(neighbors.size()), GRAPH_PO);
|
||||
std::min(static_cast<uint32_t>(neighbors.size()), prefetch_offset);
|
||||
uint32_t unvisited_count = 0;
|
||||
uint32_t i = 0;
|
||||
|
||||
@@ -200,8 +198,9 @@ template <typename EntityType, typename MemBlockType, typename FilterFn>
|
||||
void dual_heap_greedy_search(const EntityType &entity, VamanaContext *ctx,
|
||||
VamanaDistCalculator &dc, node_id_t entry_point,
|
||||
FilterFn &&filter) {
|
||||
static constexpr uint32_t PREFETCH_BATCH = 2;
|
||||
static constexpr uint32_t PREFETCH_STEP = 2;
|
||||
const uint32_t prefetch_offset = ctx->po();
|
||||
const uint32_t prefetch_lines =
|
||||
ctx->pl() > 0 ? ctx->pl() : (entity.vector_size() + 63) / 64;
|
||||
|
||||
uint32_t buf_capacity = entity.max_degree();
|
||||
std::vector<node_id_t> neighbor_ids(buf_capacity);
|
||||
@@ -273,9 +272,12 @@ void dual_heap_greedy_search(const EntityType &entity, VamanaContext *ctx,
|
||||
neighbor_vec_blocks);
|
||||
if (ailego_unlikely(ret != 0)) break;
|
||||
|
||||
for (uint32_t i = 0;
|
||||
i < std::min(PREFETCH_BATCH * PREFETCH_STEP, unvisited_count); ++i) {
|
||||
ailego_prefetch(neighbor_vec_blocks[i].data());
|
||||
for (uint32_t i = 0; i < std::min(prefetch_offset, unvisited_count); ++i) {
|
||||
const char *base =
|
||||
static_cast<const char *>(neighbor_vec_blocks[i].data());
|
||||
for (uint32_t cl = 0; cl < prefetch_lines; ++cl) {
|
||||
ailego_prefetch(base + cl * 64);
|
||||
}
|
||||
}
|
||||
|
||||
// Batch distance computation (reuse pre-allocated buffers).
|
||||
@@ -316,17 +318,8 @@ void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
|
||||
const IndexFilter &index_filter =
|
||||
static_cast<const IndexContext *>(ctx)->filter();
|
||||
|
||||
// Number of cache lines per vector (e.g. 2 for dim=128).
|
||||
// Used by both the fallback candidates/filter path and the fast helpers.
|
||||
uint32_t prefetch_lines = (dc.dimension() + 63) / 64;
|
||||
if constexpr (std::is_same_v<EntityType, VamanaContiguousStreamerEntity>) {
|
||||
// Contiguous flat array stride is already 64B-aligned. Use it so that
|
||||
// prefetch does not overshoot into the next vector.
|
||||
size_t stride = entity.vector_stride();
|
||||
if (stride > 0) {
|
||||
prefetch_lines = static_cast<uint32_t>(stride / 64);
|
||||
}
|
||||
}
|
||||
const uint32_t prefetch_lines =
|
||||
ctx->pl() > 0 ? ctx->pl() : (entity.vector_size() + 63) / 64;
|
||||
|
||||
if (!use_pool || index_filter.is_valid()) {
|
||||
// Fallback path used by add_node (use_pool=false) and filtered search.
|
||||
@@ -362,12 +355,12 @@ void VamanaAlgorithm<EntityType>::greedy_search(node_id_t entry_point,
|
||||
if (avx2_ok) {
|
||||
auto &bpool = ctx->block_pool();
|
||||
fast_greedy_search(entity, bpool, visit, dc, topk_v, ef_v, entry_point,
|
||||
prefetch_lines);
|
||||
prefetch_lines, ctx->po());
|
||||
copy_pool_to_topk(bpool, topk_heap);
|
||||
} else {
|
||||
auto &lpool = ctx->pool();
|
||||
fast_greedy_search(entity, lpool, visit, dc, topk_v, ef_v, entry_point,
|
||||
prefetch_lines);
|
||||
prefetch_lines, ctx->po());
|
||||
copy_pool_to_topk(lpool, topk_heap);
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -119,6 +119,12 @@ int VamanaContext::update(const ailego::Params ¶ms) {
|
||||
params.get(PARAM_VAMANA_STREAMER_EF, &ef);
|
||||
ef_ = ef;
|
||||
topk_heap_.limit(std::max(topk_, ef_));
|
||||
uint32_t po = po_;
|
||||
params.get(PARAM_VAMANA_STREAMER_PO, &po);
|
||||
po_ = po;
|
||||
uint32_t pl = pl_;
|
||||
params.get(PARAM_VAMANA_STREAMER_PL, &pl);
|
||||
pl_ = pl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -177,6 +177,21 @@ class VamanaContext : public IndexContext {
|
||||
inline uint32_t ef() const {
|
||||
return ef_;
|
||||
}
|
||||
inline void set_po(uint32_t v) {
|
||||
po_ = v;
|
||||
}
|
||||
|
||||
inline uint32_t po() const {
|
||||
return po_;
|
||||
}
|
||||
|
||||
inline void set_pl(uint32_t v) {
|
||||
pl_ = v;
|
||||
}
|
||||
|
||||
inline uint32_t pl() const {
|
||||
return pl_;
|
||||
}
|
||||
inline void set_max_scan_ratio(float v) {
|
||||
max_scan_ratio_ = v;
|
||||
}
|
||||
@@ -296,6 +311,8 @@ class VamanaContext : public IndexContext {
|
||||
uint32_t reserve_max_doc_cnt_{kMinReserveDocCnt};
|
||||
uint32_t topk_{0};
|
||||
uint32_t ef_{VamanaEntity::kDefaultEf};
|
||||
uint32_t po_{8};
|
||||
uint32_t pl_{0};
|
||||
float max_scan_ratio_{VamanaEntity::kDefaultScanRatio};
|
||||
size_t max_scan_limit_{VamanaEntity::kDefaultMaxScanLimit};
|
||||
size_t min_scan_limit_{VamanaEntity::kDefaultMinScanLimit};
|
||||
|
||||
@@ -48,6 +48,8 @@ static const std::string PARAM_VAMANA_STREAMER_ALPHA(
|
||||
static const std::string PARAM_VAMANA_STREAMER_MAX_OCCLUSION_SIZE(
|
||||
"proxima.vamana.streamer.max_occlusion_size");
|
||||
static const std::string PARAM_VAMANA_STREAMER_EF("proxima.vamana.streamer.ef");
|
||||
static const std::string PARAM_VAMANA_STREAMER_PO("proxima.vamana.streamer.po");
|
||||
static const std::string PARAM_VAMANA_STREAMER_PL("proxima.vamana.streamer.pl");
|
||||
static const std::string PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD(
|
||||
"proxima.vamana.streamer.brute_force_threshold");
|
||||
static const std::string PARAM_VAMANA_STREAMER_MAX_SCAN_RATIO(
|
||||
|
||||
Reference in New Issue
Block a user