fix: Replace VLAs with std::vector for MSVC compatibility and stack safety (#190)
This commit is contained in:
@@ -14,6 +14,7 @@
|
||||
#include "hnsw_algorithm.h"
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
|
||||
namespace zvec {
|
||||
@@ -132,13 +133,13 @@ void HnswAlgorithm::select_entry_point(level_t level, node_id_t *entry_point,
|
||||
|
||||
bool find_closer = false;
|
||||
|
||||
float dists[size];
|
||||
const void *neighbor_vecs[size];
|
||||
std::vector<float> dists(size);
|
||||
std::vector<const void *> neighbor_vecs(size);
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
|
||||
}
|
||||
|
||||
dc.batch_dist(neighbor_vecs, size, dists);
|
||||
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
dist_t cur_dist = dists[i];
|
||||
@@ -213,7 +214,7 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
|
||||
node_id_t neighbor_ids[neighbors.size()];
|
||||
std::vector<node_id_t> neighbor_ids(neighbors.size());
|
||||
uint32_t size = 0;
|
||||
for (uint32_t i = 0; i < neighbors.size(); ++i) {
|
||||
node_id_t node = neighbors[i];
|
||||
@@ -231,7 +232,7 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
|
||||
}
|
||||
|
||||
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
|
||||
int ret = entity.get_vector(neighbor_ids, size, neighbor_vec_blocks);
|
||||
int ret = entity.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_vector())++;
|
||||
}
|
||||
@@ -247,14 +248,14 @@ void HnswAlgorithm::search_neighbors(level_t level, node_id_t *entry_point,
|
||||
}
|
||||
// done
|
||||
|
||||
float dists[size];
|
||||
const void *neighbor_vecs[size];
|
||||
std::vector<float> dists(size);
|
||||
std::vector<const void *> neighbor_vecs(size);
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
neighbor_vecs[i] = neighbor_vec_blocks[i].data();
|
||||
}
|
||||
|
||||
dc.batch_dist(neighbor_vecs, size, dists);
|
||||
dc.batch_dist(neighbor_vecs.data(), size, dists.data());
|
||||
|
||||
for (uint32_t i = 0; i < size; ++i) {
|
||||
node_id_t node = neighbor_ids[i];
|
||||
@@ -336,7 +337,7 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
|
||||
node_id_t neighbor_ids[neighbors.size()];
|
||||
std::vector<node_id_t> neighbor_ids(neighbors.size());
|
||||
uint32_t size = 0;
|
||||
for (uint32_t i = 0; i < neighbors.size(); ++i) {
|
||||
node_id_t node = neighbors[i];
|
||||
@@ -354,7 +355,8 @@ void HnswAlgorithm::expand_neighbors_by_group(TopkHeap &topk,
|
||||
}
|
||||
|
||||
std::vector<IndexStorage::MemoryBlock> neighbor_vec_blocks;
|
||||
int ret = entity.get_vector(neighbor_ids, size, neighbor_vec_blocks);
|
||||
int ret =
|
||||
entity.get_vector(neighbor_ids.data(), size, neighbor_vec_blocks);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_vector())++;
|
||||
}
|
||||
|
||||
@@ -172,8 +172,8 @@ int64_t HnswEntity::dump_vectors(
|
||||
|
||||
size_t padding_size = AlignSize(vector_dump_size) - vector_dump_size;
|
||||
|
||||
char padding[padding_size];
|
||||
memset(padding, 0, sizeof(padding));
|
||||
std::vector<char> padding(padding_size);
|
||||
memset(padding.data(), 0, sizeof(char) * padding_size);
|
||||
const void *data = nullptr;
|
||||
uint32_t crc = 0U;
|
||||
size_t vecs_size = 0UL;
|
||||
@@ -198,12 +198,12 @@ int64_t HnswEntity::dump_vectors(
|
||||
continue;
|
||||
}
|
||||
|
||||
len = dumper->write(padding, padding_size);
|
||||
len = dumper->write(padding.data(), padding_size);
|
||||
if (len != padding_size) {
|
||||
LOG_ERROR("Dump vectors failed, write=%zu expect=%zu", len, padding_size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(padding, padding_size, crc);
|
||||
crc = ailego::Crc32c::Hash(padding.data(), padding_size, crc);
|
||||
vecs_size += padding_size;
|
||||
}
|
||||
|
||||
@@ -224,7 +224,7 @@ int64_t HnswEntity::dump_graph_neighbors(
|
||||
graph_meta.reserve(doc_cnt());
|
||||
size_t offset = 0;
|
||||
uint32_t crc = 0;
|
||||
node_id_t mapping[l0_neighbor_cnt()];
|
||||
std::vector<node_id_t> mapping(l0_neighbor_cnt());
|
||||
|
||||
uint32_t min_neighbor_count = 10000;
|
||||
uint32_t max_neighbor_count = 0;
|
||||
@@ -253,7 +253,7 @@ int64_t HnswEntity::dump_graph_neighbors(
|
||||
for (node_id_t i = 0; i < neighbors.size(); ++i) {
|
||||
mapping[i] = neighbor_mapping[neighbors[i]];
|
||||
}
|
||||
data = mapping;
|
||||
data = mapping.data();
|
||||
}
|
||||
if (dumper->write(data, size) != size) {
|
||||
LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id, size);
|
||||
@@ -303,7 +303,7 @@ int64_t HnswEntity::dump_upper_neighbors(
|
||||
hnsw_meta.reserve(doc_cnt());
|
||||
size_t offset = 0;
|
||||
uint32_t crc = 0;
|
||||
node_id_t buffer[upper_neighbor_cnt() + 1];
|
||||
std::vector<node_id_t> buffer(upper_neighbor_cnt() + 1);
|
||||
for (node_id_t id = 0; id < doc_cnt(); ++id) {
|
||||
node_id_t new_id = reorder_mapping.empty() ? id : reorder_mapping[id];
|
||||
auto level = get_level(new_id);
|
||||
@@ -318,7 +318,7 @@ int64_t HnswEntity::dump_upper_neighbors(
|
||||
ailego_assert_with(!!neighbors.data, "invalid neighbors");
|
||||
ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level),
|
||||
"invalid neighbors");
|
||||
memset(buffer, 0, sizeof(buffer));
|
||||
memset(buffer.data(), 0, sizeof(node_id_t) * buffer.size());
|
||||
buffer[0] = neighbors.size();
|
||||
if (neighbor_mapping.empty()) {
|
||||
memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t));
|
||||
@@ -327,13 +327,15 @@ int64_t HnswEntity::dump_upper_neighbors(
|
||||
buffer[i + 1] = neighbor_mapping[neighbors[i]];
|
||||
}
|
||||
}
|
||||
if (dumper->write(buffer, sizeof(buffer)) != sizeof(buffer)) {
|
||||
if (dumper->write(buffer.data(), sizeof(node_id_t) * buffer.size()) !=
|
||||
sizeof(node_id_t) * buffer.size()) {
|
||||
LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id,
|
||||
sizeof(buffer));
|
||||
sizeof(node_id_t) * buffer.size());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(buffer, sizeof(buffer), crc);
|
||||
offset += sizeof(buffer);
|
||||
crc = ailego::Crc32c::Hash(buffer.data(),
|
||||
sizeof(node_id_t) * buffer.size(), crc);
|
||||
offset += sizeof(node_id_t) * buffer.size();
|
||||
}
|
||||
}
|
||||
size_t padding_size = 0;
|
||||
|
||||
@@ -80,8 +80,8 @@ int HnswStreamerEntity::cleanup() {
|
||||
int HnswStreamerEntity::update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
|
||||
char buffer[neighbor_size_];
|
||||
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer);
|
||||
std::vector<char> buffer(neighbor_size_);
|
||||
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer.data());
|
||||
hd->neighbor_cnt = neighbors.size();
|
||||
size_t i = 0;
|
||||
for (; i < neighbors.size(); ++i) {
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "hnsw_sparse_algorithm.h"
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
|
||||
namespace zvec {
|
||||
@@ -208,7 +209,7 @@ void HnswSparseAlgorithm::search_neighbors(level_t level,
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
|
||||
node_id_t neighbor_ids[neighbors.size()];
|
||||
std::vector<node_id_t> neighbor_ids(neighbors.size());
|
||||
uint32_t size = 0;
|
||||
for (uint32_t i = 0; i < neighbors.size(); ++i) {
|
||||
node_id_t node = neighbors[i];
|
||||
@@ -226,7 +227,8 @@ void HnswSparseAlgorithm::search_neighbors(level_t level,
|
||||
}
|
||||
|
||||
std::vector<IndexStorage::MemoryBlock> neighbor_block_vecs;
|
||||
int ret = entity.get_vector_metas(neighbor_ids, size, neighbor_block_vecs);
|
||||
int ret =
|
||||
entity.get_vector_metas(neighbor_ids.data(), size, neighbor_block_vecs);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_vector())++;
|
||||
}
|
||||
@@ -337,7 +339,7 @@ void HnswSparseAlgorithm::expand_neighbors_by_group(
|
||||
(*ctx->mutable_stats_get_neighbors())++;
|
||||
}
|
||||
|
||||
node_id_t neighbor_ids[neighbors.size()];
|
||||
std::vector<node_id_t> neighbor_ids(neighbors.size());
|
||||
uint32_t size = 0;
|
||||
for (uint32_t i = 0; i < neighbors.size(); ++i) {
|
||||
node_id_t node = neighbors[i];
|
||||
@@ -355,8 +357,8 @@ void HnswSparseAlgorithm::expand_neighbors_by_group(
|
||||
}
|
||||
|
||||
std::vector<IndexStorage::MemoryBlock> neighbor_block_vecs;
|
||||
int ret =
|
||||
entity.get_vector_metas(neighbor_ids, size, neighbor_block_vecs);
|
||||
int ret = entity.get_vector_metas(neighbor_ids.data(), size,
|
||||
neighbor_block_vecs);
|
||||
if (ailego_unlikely(ctx->debugging())) {
|
||||
(*ctx->mutable_stats_get_vector())++;
|
||||
}
|
||||
|
||||
@@ -299,7 +299,7 @@ int64_t HnswSparseEntity::dump_graph_neighbors(
|
||||
graph_meta.reserve(doc_cnt());
|
||||
size_t offset = 0;
|
||||
uint32_t crc = 0;
|
||||
node_id_t mapping[l0_neighbor_cnt()];
|
||||
std::vector<node_id_t> mapping(l0_neighbor_cnt());
|
||||
|
||||
uint32_t min_neighbor_count = 10000;
|
||||
uint32_t max_neighbor_count = 0;
|
||||
@@ -328,7 +328,7 @@ int64_t HnswSparseEntity::dump_graph_neighbors(
|
||||
for (node_id_t i = 0; i < neighbors.size(); ++i) {
|
||||
mapping[i] = neighbor_mapping[neighbors[i]];
|
||||
}
|
||||
data = mapping;
|
||||
data = mapping.data();
|
||||
}
|
||||
if (dumper->write(data, size) != size) {
|
||||
LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id, size);
|
||||
@@ -380,7 +380,7 @@ int64_t HnswSparseEntity::dump_upper_neighbors(
|
||||
hnsw_meta.reserve(doc_cnt());
|
||||
size_t offset = 0;
|
||||
uint32_t crc = 0;
|
||||
node_id_t buffer[upper_neighbor_cnt() + 1];
|
||||
std::vector<node_id_t> buffer(upper_neighbor_cnt() + 1);
|
||||
for (node_id_t id = 0; id < doc_cnt(); ++id) {
|
||||
node_id_t new_id = reorder_mapping.empty() ? id : reorder_mapping[id];
|
||||
auto level = get_level(new_id);
|
||||
@@ -395,7 +395,7 @@ int64_t HnswSparseEntity::dump_upper_neighbors(
|
||||
ailego_assert_with(!!neighbors.data, "invalid neighbors");
|
||||
ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level),
|
||||
"invalid neighbors");
|
||||
memset(buffer, 0, sizeof(buffer));
|
||||
memset(buffer.data(), 0, sizeof(node_id_t) * buffer.size());
|
||||
buffer[0] = neighbors.size();
|
||||
if (neighbor_mapping.empty()) {
|
||||
memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t));
|
||||
@@ -404,13 +404,15 @@ int64_t HnswSparseEntity::dump_upper_neighbors(
|
||||
buffer[i + 1] = neighbor_mapping[neighbors[i]];
|
||||
}
|
||||
}
|
||||
if (dumper->write(buffer, sizeof(buffer)) != sizeof(buffer)) {
|
||||
if (dumper->write(buffer.data(), sizeof(node_id_t) * buffer.size()) !=
|
||||
sizeof(node_id_t) * buffer.size()) {
|
||||
LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id,
|
||||
sizeof(buffer));
|
||||
sizeof(node_id_t) * buffer.size());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
crc = ailego::Crc32c::Hash(buffer, sizeof(buffer), crc);
|
||||
offset += sizeof(buffer);
|
||||
crc = ailego::Crc32c::Hash(buffer.data(),
|
||||
sizeof(node_id_t) * buffer.size(), crc);
|
||||
offset += sizeof(node_id_t) * buffer.size();
|
||||
}
|
||||
}
|
||||
size_t padding_size = 0;
|
||||
|
||||
@@ -84,8 +84,8 @@ int HnswSparseStreamerEntity::cleanup() {
|
||||
int HnswSparseStreamerEntity::update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) {
|
||||
char buffer[neighbor_size_];
|
||||
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer);
|
||||
std::vector<char> buffer(neighbor_size_);
|
||||
NeighborsHeader *hd = reinterpret_cast<NeighborsHeader *>(buffer.data());
|
||||
hd->neighbor_cnt = neighbors.size();
|
||||
size_t i = 0;
|
||||
for (; i < neighbors.size(); ++i) {
|
||||
|
||||
@@ -595,7 +595,7 @@ int IVFEntity::search(size_t inverted_list_id, const void *query,
|
||||
|
||||
const void *data = nullptr;
|
||||
const size_t block_vecs = header_.block_vector_count;
|
||||
float distances[block_vecs];
|
||||
std::vector<float> distances(block_vecs);
|
||||
const size_t batch_size = kBatchBlocks;
|
||||
const size_t block_size = header_.block_size;
|
||||
const auto norm_val = this->inverted_list_normalize_value(inverted_list_id);
|
||||
@@ -639,7 +639,7 @@ int IVFEntity::search(size_t inverted_list_id, const void *query,
|
||||
|
||||
const void *block_data = static_cast<const char *>(data) + b * block_size;
|
||||
calculator_->query_features_distance(query, block_data, vecs_count,
|
||||
distances);
|
||||
distances.data());
|
||||
|
||||
*(context_stats->mutable_dist_calced_count()) += vecs_count;
|
||||
|
||||
@@ -669,7 +669,7 @@ int IVFEntity::search(size_t inverted_list_id, const void *query,
|
||||
|
||||
const void *data = nullptr;
|
||||
const size_t block_vecs = header_.block_vector_count;
|
||||
float distances[block_vecs];
|
||||
std::vector<float> distances(block_vecs);
|
||||
const size_t batch_size = kBatchBlocks;
|
||||
const size_t block_size = header_.block_size;
|
||||
const auto norm_val = this->inverted_list_normalize_value(inverted_list_id);
|
||||
@@ -700,7 +700,7 @@ int IVFEntity::search(size_t inverted_list_id, const void *query,
|
||||
auto block_keys = keys + b * block_vecs;
|
||||
const void *block_data = static_cast<const char *>(data) + b * block_size;
|
||||
calculator_->query_features_distance(query, block_data, vecs_count,
|
||||
distances);
|
||||
distances.data());
|
||||
for (size_t k = 0; k < vecs_count; ++k) {
|
||||
if (block_keys[k] != kInvalidKey) {
|
||||
uint32_t id = list_meta->id_offset + (i + b) * block_vecs + k;
|
||||
|
||||
Reference in New Issue
Block a user