refactor: flat param string cleanup (#28)

* refactor: flat param string cleanup

* fix
This commit is contained in:
Jalin Wang
2026-01-26 14:49:35 +08:00
committed by GitHub
parent 595161a6d2
commit bc90952a01
12 changed files with 49 additions and 73 deletions
+6 -6
View File
@@ -25,7 +25,7 @@ int FlatBuilder<BATCH_SIZE>::init(const IndexMeta &meta,
// Set the major order
bool column_major_order = false;
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order)) {
if (params.get(PARAM_FLAT_COLUMN_MAJOR_ORDER, &column_major_order)) {
meta_.set_major_order(column_major_order ? IndexMeta::MO_COLUMN
: IndexMeta::MO_ROW);
}
@@ -71,7 +71,7 @@ int FlatBuilder<BATCH_SIZE>::init(const IndexMeta &meta,
std::string tag = std::to_string(BATCH_SIZE);
ailego::Params searcher_params;
searcher_params.set(FLAT_SEARCHER_BATCH_SIZE, BATCH_SIZE);
searcher_params.set(PARAM_FLAT_BATCH_SIZE, BATCH_SIZE);
meta_.set_searcher("FlatSearcher" + tag, 0, searcher_params);
meta_.set_builder("FlatBuilder" + tag, 0, params);
return 0;
@@ -151,7 +151,7 @@ int FlatBuilder<BATCH_SIZE>::write_keys(const std::vector<uint64_t> &keys,
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
return dumper->append(FLAT_SEGMENT_KEYS_SEG_ID, keys_size, keys_padding_size,
0);
}
@@ -179,7 +179,7 @@ int FlatBuilder<BATCH_SIZE>::write_mapping(const std::vector<uint64_t> &keys,
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
return dumper->append(FLAT_SEGMENT_MAPPING_SEG_ID, mapping_size,
mapping_padding_size, 0);
}
@@ -236,7 +236,7 @@ int FlatBuilder<BATCH_SIZE>::write_column_index(IndexDumper *dumper,
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
features_padding_size, 0);
}
@@ -271,7 +271,7 @@ int FlatBuilder<BATCH_SIZE>::write_row_index(IndexDumper *dumper,
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
features_padding_size, 0);
}
+6 -6
View File
@@ -27,10 +27,10 @@ IndexProvider::Pointer FlatSearcher<BATCH_SIZE>::create_provider(void) const {
std::lock_guard<std::mutex> lock(mapping_mutex_);
if (mapping_.empty()) {
auto mapping_segment = container_->get(PARAM_FLAT_SEGMENT_MAPPING);
auto mapping_segment = container_->get(FLAT_SEGMENT_MAPPING_SEG_ID);
if (!mapping_segment) {
LOG_ERROR("Failed to fetch segment %s",
PARAM_FLAT_SEGMENT_MAPPING.c_str());
FLAT_SEGMENT_MAPPING_SEG_ID.c_str());
return nullptr;
}
@@ -115,14 +115,14 @@ int FlatSearcher<BATCH_SIZE>::load(IndexStorage::Pointer cntr,
}
}
auto keys_segment = cntr->get(PARAM_FLAT_SEGMENT_KEYS);
auto keys_segment = cntr->get(FLAT_SEGMENT_KEYS_SEG_ID);
if (!keys_segment) {
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
LOG_ERROR("Failed to fetch segment %s", FLAT_SEGMENT_KEYS_SEG_ID.c_str());
return IndexError_NoExist;
}
features_segment_ = cntr->get(PARAM_FLAT_SEGMENT_FEATURES);
features_segment_ = cntr->get(FLAT_SEGMENT_FEATURES_SEG_ID);
if (!features_segment_) {
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
LOG_ERROR("Failed to fetch segment %s", FLAT_SEGMENT_KEYS_SEG_ID.c_str());
return IndexError_NoExist;
}
+3 -3
View File
@@ -32,8 +32,8 @@ class FlatSearcher : public IndexSearcher {
//! Initialize Searcher
int init(const ailego::Params &index_params) override {
params_ = index_params;
read_block_size_ = FLAT_READ_BLOCK_SIZE;
index_params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
read_block_size_ = FLAT_DEFAULT_READ_BLOCK_SIZE;
index_params.get(PARAM_FLAT_READ_BLOCK_SIZE, &read_block_size_);
return 0;
}
@@ -165,7 +165,7 @@ class FlatSearcher : public IndexSearcher {
const uint64_t *keys_{nullptr};
std::map<key_t, node_id_t> key_id_mapping_;
uint32_t magic_{IndexContext::GenerateMagic()};
uint32_t read_block_size_{FLAT_READ_BLOCK_SIZE};
uint32_t read_block_size_{FLAT_DEFAULT_READ_BLOCK_SIZE};
bool column_major_order_{false};
IndexMeta meta_{};
IndexStorage::Pointer container_{};
+4 -4
View File
@@ -57,7 +57,7 @@ int FlatStreamer<BATCH_SIZE>::init(const IndexMeta &imeta,
}
// 参数设置
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order_)) {
if (params.get(PARAM_FLAT_COLUMN_MAJOR_ORDER, &column_major_order_)) {
meta_.set_major_order(column_major_order_ ? IndexMeta::MO_COLUMN
: IndexMeta::MO_ROW);
}
@@ -99,9 +99,9 @@ int FlatStreamer<BATCH_SIZE>::init(const IndexMeta &imeta,
return IndexError_InvalidArgument;
}
read_block_size_ = FLAT_READ_BLOCK_SIZE;
params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
params.get(FLAT_SEARCHER_USE_ID_MAP, &use_key_info_map_);
read_block_size_ = FLAT_DEFAULT_READ_BLOCK_SIZE;
params.get(PARAM_FLAT_READ_BLOCK_SIZE, &read_block_size_);
params.get(PARAM_FLAT_USE_ID_MAP, &use_key_info_map_);
// entity init
uint32_t block_vector_count = kDefaultBlockVecCount;
@@ -140,7 +140,7 @@ class FlatStreamerDumper {
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
features_padding_size, 0);
}
@@ -177,7 +177,7 @@ class FlatStreamerDumper {
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
features_padding_size, 0);
}
@@ -200,8 +200,8 @@ class FlatStreamerDumper {
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
0);
return dumper->append(FLAT_SEGMENT_KEYS_SEG_ID, keys_size,
keys_padding_size, 0);
}
int write_mapping(const std::vector<uint64_t> &keys, IndexDumper *dumper) {
@@ -230,7 +230,7 @@ class FlatStreamerDumper {
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
return dumper->append(FLAT_SEGMENT_MAPPING_SEG_ID, mapping_size,
mapping_padding_size, 0);
}
@@ -688,7 +688,7 @@ int FlatStreamerEntity::load_storage(IndexStorage::Pointer storage) {
size_t index_size = hd_segment->capacity();
for (size_t i = 1; i <= meta_.segment_count; ++i) {
std::string segment_id =
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, i);
auto seg = storage->get(segment_id);
if (!seg || seg->data_size() < meta_.header.block_size) {
LOG_ERROR("Failed to get segment %s, or invalid segment size",
@@ -756,7 +756,7 @@ int FlatStreamerEntity::alloc_segment(void) {
}
std::string segment_id =
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, index);
ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, index);
size_t size =
ailego_align(meta_.segment_size, ailego::MemoryHelper::PageSize());
auto segment = storage_->get(segment_id);
@@ -219,7 +219,7 @@ class FlatStreamerEntity {
const IndexStorage::Segment::Pointer get_segment(size_t index) const {
for (size_t i = segments_.size(); i <= index; ++i) {
auto segment_id =
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, i);
auto segment = storage_->get(segment_id);
if (!segment) {
LOG_ERROR("Failed to get segment %s", segment_id.c_str());
+12 -23
View File
@@ -25,32 +25,21 @@ namespace zvec {
namespace core {
//! The default size of reading a block
static constexpr uint32_t FLAT_READ_BLOCK_SIZE = 4 * 1024 * 1024;
static const std::string FLAT_LINEAR_META_SEG_ID = "linear.linear_meta";
static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID =
"linear.linear_list_head";
static constexpr uint32_t FLAT_DEFAULT_READ_BLOCK_SIZE = 4 * 1024 * 1024;
static const std::string FLAT_LINEAR_META_SEG_ID = "flat.linear_meta";
static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID = "flat.linear_list_head";
// reducer params
static const std::string PARAM_FLAT_REDUCER_WORKING_PATH(
"proxima.linear.reducer.working_path");
static const std::string PARAM_FLAT_REDUCER_NUM_OF_ADD_THREADS(
"proxima.linear.reducer.num_of_add_threads");
static const std::string PARAM_FLAT_REDUCER_INDEX_NAME(
"proxima.linear.reducer.index_name");
static const std::string FLAT_SEGMENT_KEYS_SEG_ID("flat.keys");
static const std::string FLAT_SEGMENT_FEATURES_SEG_ID("flat.features");
static const std::string FLAT_SEGMENT_MAPPING_SEG_ID("flat.mapping");
// index params
static const std::string PARAM_FLAT_SEGMENT_KEYS("linear.keys");
static const std::string PARAM_FLAT_SEGMENT_FEATURES("linear.features");
static const std::string PARAM_FLAT_SEGMENT_MAPPING("linear.mapping");
static const std::string FLAT_BUILDER_COLUMN_MAJOR_ORDER(
"proxima.linear.builder.column_major_order");
static const std::string FLAT_SEARCHER_BATCH_SIZE(
"proxima.linear.searcher.batch_size");
static const std::string FLAT_SEARCHER_READ_BLOCK_SIZE(
"proxima.linear.searcher.read_block_size");
static const std::string FLAT_SEARCHER_USE_ID_MAP(
"proxima.linear.searcher.use_id_map");
static const std::string PARAM_FLAT_COLUMN_MAJOR_ORDER(
"proxima.flat.column_major_order");
static const std::string PARAM_FLAT_BATCH_SIZE("proxima.flat.batch_size");
static const std::string PARAM_FLAT_READ_BLOCK_SIZE(
"proxima.flat.read_block_size");
static const std::string PARAM_FLAT_USE_ID_MAP("proxima.flat.use_id_map");
//! Determines if a number is equal to two to the power of n.
template <size_t K>