fix: Replace VLAs with std::vector for MSVC compatibility and stack safety (#190)

This commit is contained in:
feihongxu0824
2026-03-03 10:07:21 +08:00
committed by GitHub
parent 9eb68ee061
commit 24d877da5e
38 changed files with 294 additions and 265 deletions
+12 -10
View File
@@ -14,6 +14,7 @@
#include "hnsw_algorithm.h"
#include <chrono>
#include <iostream>
#include <vector>
#include <ailego/internal/cpu_features.h>
namespace zvec {
@@ -132,13 +133,13 @@ void HnswAlgorithm::select_entry_point(level_t level, node_id_t *entry_point,
bool find_closer = false;
float dists[size];
const void *neighbor_vecs[size];
std::vector<float> dists(size);
std::vector<const void *> neighbor_vecs(size);
for (uint32_t i = 0; i < size; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
}
dc.batch_dist(neighbor_vecs, size, dists);
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
for (uint32_t i = 0; i < size; ++i) {
dist_t cur_dist = dists[i];
@@ -213,7 +214,7 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
(*ctx->mutable_stats_get_neighbors())++;
}
node_id_t neighbor_ids[neighbors.size()];
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
@@ -231,7 +232,7 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
}
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
int ret = entity.get_vector(neighbor_ids, size, neighbor_vec_blocks);
int ret = entity.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
@@ -247,14 +248,14 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
}
// done
float dists[size];
const void *neighbor_vecs[size];
std::vector<float> dists(size);
std::vector<const void *> neighbor_vecs(size);
for (uint32_t i = 0; i < size; ++i) {
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
}
dc.batch_dist(neighbor_vecs, size, dists);
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
for (uint32_t i = 0; i < size; ++i) {
node_id_t node = neighbor_ids[i];
@@ -336,7 +337,7 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
(*ctx->mutable_stats_get_neighbors())++;
}
node_id_t neighbor_ids[neighbors.size()];
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
@@ -354,7 +355,8 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
}
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
int ret = entity.get_vector(neighbor_ids, size, neighbor_vec_blocks);
int ret =
entity.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
+14 -12
View File
@@ -172,8 +172,8 @@ int64_t HnswEntity::dump_vectors(
size_t padding_size = AlignSize(vector_dump_size) - vector_dump_size;
char padding[padding_size];
memset(padding, 0, sizeof(padding));
std::vector<char> padding(padding_size);
memset(padding.data(), 0, sizeof(char) * padding_size);
const void *data = nullptr;
uint32_t crc = 0U;
size_t vecs_size = 0UL;
@@ -198,12 +198,12 @@ int64_t HnswEntity::dump_vectors(
continue;
}
len = dumper->write(padding, padding_size);
len = dumper->write(padding.data(), padding_size);
if (len != padding_size) {
LOG_ERROR("Dump vectors failed, write=%zu expect=%zu", len, padding_size);
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(padding, padding_size, crc);
crc = ailego::Crc32c::Hash(padding.data(), padding_size, crc);
vecs_size += padding_size;
}
@@ -224,7 +224,7 @@ int64_t HnswEntity::dump_graph_neighbors(
graph_meta.reserve(doc_cnt());
size_t offset = 0;
uint32_t crc = 0;
node_id_t mapping[l0_neighbor_cnt()];
std::vector<node_id_t> mapping(l0_neighbor_cnt());
uint32_t min_neighbor_count = 10000;
uint32_t max_neighbor_count = 0;
@@ -253,7 +253,7 @@ int64_t HnswEntity::dump_graph_neighbors(
for (node_id_t i = 0; i < neighbors.size(); ++i) {
mapping[i] = neighbor_mapping[neighbors[i]];
}
data = mapping;
data = mapping.data();
}
if (dumper->write(data, size) != size) {
LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id, size);
@@ -303,7 +303,7 @@ int64_t HnswEntity::dump_upper_neighbors(
hnsw_meta.reserve(doc_cnt());
size_t offset = 0;
uint32_t crc = 0;
node_id_t buffer[upper_neighbor_cnt() + 1];
std::vector<node_id_t> buffer(upper_neighbor_cnt() + 1);
for (node_id_t id = 0; id < doc_cnt(); ++id) {
node_id_t new_id = reorder_mapping.empty() ? id : reorder_mapping[id];
auto level = get_level(new_id);
@@ -318,7 +318,7 @@ int64_t HnswEntity::dump_upper_neighbors(
ailego_assert_with(!!neighbors.data, "invalid neighbors");
ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level),
"invalid neighbors");
memset(buffer, 0, sizeof(buffer));
memset(buffer.data(), 0, sizeof(node_id_t) * buffer.size());
buffer[0] = neighbors.size();
if (neighbor_mapping.empty()) {
memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t));
@@ -327,13 +327,15 @@ int64_t HnswEntity::dump_upper_neighbors(
buffer[i + 1] = neighbor_mapping[neighbors[i]];
}
}
if (dumper->write(buffer, sizeof(buffer)) != sizeof(buffer)) {
if (dumper->write(buffer.data(), sizeof(node_id_t) * buffer.size()) !=
sizeof(node_id_t) * buffer.size()) {
LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id,
sizeof(buffer));
sizeof(node_id_t) * buffer.size());
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(buffer, sizeof(buffer), crc);
offset += sizeof(buffer);
crc = ailego::Crc32c::Hash(buffer.data(),
sizeof(node_id_t) * buffer.size(), crc);
offset += sizeof(node_id_t) * buffer.size();
}
}
size_t padding_size = 0;
@@ -80,8 +80,8 @@ int HnswStreamerEntity::cleanup() {
int HnswStreamerEntity::update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
char buffer[neighbor_size_];
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer);
std::vector<char> buffer(neighbor_size_);
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer.data());
hd->neighbor_cnt = neighbors.size();
size_t i = 0;
for (; i < neighbors.size(); ++i) {
@@ -15,6 +15,7 @@
#include "hnsw_sparse_algorithm.h"
#include <chrono>
#include <iostream>
#include <vector>
#include <ailego/internal/cpu_features.h>
namespace zvec {
@@ -208,7 +209,7 @@ void HnswSparseAlgorithm::search_neighbors(level_t level,
(*ctx->mutable_stats_get_neighbors())++;
}
node_id_t neighbor_ids[neighbors.size()];
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
@@ -226,7 +227,8 @@ void HnswSparseAlgorithm::search_neighbors(level_t level,
}
std::vector<IndexStorage::MemoryBlock> neighbor_block_vecs;
int ret = entity.get_vector_metas(neighbor_ids, size, neighbor_block_vecs);
int ret =
entity.get_vector_metas(neighbor_ids.data(), size, neighbor_block_vecs);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
@@ -337,7 +339,7 @@ void HnswSparseAlgorithm::expand_neighbors_by_group(
(*ctx->mutable_stats_get_neighbors())++;
}
node_id_t neighbor_ids[neighbors.size()];
std::vector<node_id_t> neighbor_ids(neighbors.size());
uint32_t size = 0;
for (uint32_t i = 0; i < neighbors.size(); ++i) {
node_id_t node = neighbors[i];
@@ -355,8 +357,8 @@ void HnswSparseAlgorithm::expand_neighbors_by_group(
}
std::vector<IndexStorage::MemoryBlock> neighbor_block_vecs;
int ret =
entity.get_vector_metas(neighbor_ids, size, neighbor_block_vecs);
int ret = entity.get_vector_metas(neighbor_ids.data(), size,
neighbor_block_vecs);
if (ailego_unlikely(ctx->debugging())) {
(*ctx->mutable_stats_get_vector())++;
}
@@ -299,7 +299,7 @@ int64_t HnswSparseEntity::dump_graph_neighbors(
graph_meta.reserve(doc_cnt());
size_t offset = 0;
uint32_t crc = 0;
node_id_t mapping[l0_neighbor_cnt()];
std::vector<node_id_t> mapping(l0_neighbor_cnt());
uint32_t min_neighbor_count = 10000;
uint32_t max_neighbor_count = 0;
@@ -328,7 +328,7 @@ int64_t HnswSparseEntity::dump_graph_neighbors(
for (node_id_t i = 0; i < neighbors.size(); ++i) {
mapping[i] = neighbor_mapping[neighbors[i]];
}
data = mapping;
data = mapping.data();
}
if (dumper->write(data, size) != size) {
LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id, size);
@@ -380,7 +380,7 @@ int64_t HnswSparseEntity::dump_upper_neighbors(
hnsw_meta.reserve(doc_cnt());
size_t offset = 0;
uint32_t crc = 0;
node_id_t buffer[upper_neighbor_cnt() + 1];
std::vector<node_id_t> buffer(upper_neighbor_cnt() + 1);
for (node_id_t id = 0; id < doc_cnt(); ++id) {
node_id_t new_id = reorder_mapping.empty() ? id : reorder_mapping[id];
auto level = get_level(new_id);
@@ -395,7 +395,7 @@ int64_t HnswSparseEntity::dump_upper_neighbors(
ailego_assert_with(!!neighbors.data, "invalid neighbors");
ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level),
"invalid neighbors");
memset(buffer, 0, sizeof(buffer));
memset(buffer.data(), 0, sizeof(node_id_t) * buffer.size());
buffer[0] = neighbors.size();
if (neighbor_mapping.empty()) {
memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t));
@@ -404,13 +404,15 @@ int64_t HnswSparseEntity::dump_upper_neighbors(
buffer[i + 1] = neighbor_mapping[neighbors[i]];
}
}
if (dumper->write(buffer, sizeof(buffer)) != sizeof(buffer)) {
if (dumper->write(buffer.data(), sizeof(node_id_t) * buffer.size()) !=
sizeof(node_id_t) * buffer.size()) {
LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id,
sizeof(buffer));
sizeof(node_id_t) * buffer.size());
return IndexError_WriteData;
}
crc = ailego::Crc32c::Hash(buffer, sizeof(buffer), crc);
offset += sizeof(buffer);
crc = ailego::Crc32c::Hash(buffer.data(),
sizeof(node_id_t) * buffer.size(), crc);
offset += sizeof(node_id_t) * buffer.size();
}
}
size_t padding_size = 0;
@@ -84,8 +84,8 @@ int HnswSparseStreamerEntity::cleanup() {
int HnswSparseStreamerEntity::update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
char buffer[neighbor_size_];
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer);
std::vector<char> buffer(neighbor_size_);
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer.data());
hd->neighbor_cnt = neighbors.size();
size_t i = 0;
for (; i < neighbors.size(); ++i) {
+4 -4
View File
@@ -595,7 +595,7 @@ int IVFEntity::search(size_t inverted_list_id, const void *query,
const void *data = nullptr;
const size_t block_vecs = header_.block_vector_count;
float distances[block_vecs];
std::vector<float> distances(block_vecs);
const size_t batch_size = kBatchBlocks;
const size_t block_size = header_.block_size;
const auto norm_val = this->inverted_list_normalize_value(inverted_list_id);
@@ -639,7 +639,7 @@ int IVFEntity::search(size_t inverted_list_id, const void *query,
const void *block_data = static_cast<const char *>(data) + b * block_size;
calculator_->query_features_distance(query, block_data, vecs_count,
distances);
distances.data());
*(context_stats->mutable_dist_calced_count()) += vecs_count;
@@ -669,7 +669,7 @@ int IVFEntity::search(size_t inverted_list_id, const void *query,
const void *data = nullptr;
const size_t block_vecs = header_.block_vector_count;
float distances[block_vecs];
std::vector<float> distances(block_vecs);
const size_t batch_size = kBatchBlocks;
const size_t block_size = header_.block_size;
const auto norm_val = this->inverted_list_normalize_value(inverted_list_id);
@@ -700,7 +700,7 @@ int IVFEntity::search(size_t inverted_list_id, const void *query,
auto block_keys = keys + b * block_vecs;
const void *block_data = static_cast<const char *>(data) + b * block_size;
calculator_->query_features_distance(query, block_data, vecs_count,
distances);
distances.data());
for (size_t k = 0; k < vecs_count; ++k) {
if (block_keys[k] != kInvalidKey) {
uint32_t id = list_meta->id_offset + (i + b) * block_vecs + k;