From bc90952a0188db57dc85c9fe5d977bf01021a022 Mon Sep 17 00:00:00 2001 From: Jalin Wang Date: Mon, 26 Jan 2026 14:49:35 +0800 Subject: [PATCH] refactor: flat param string cleanup (#28) * refactor: flat param string cleanup * fix --- src/core/algorithm/flat/flat_builder.cc | 12 +++---- src/core/algorithm/flat/flat_searcher.cc | 12 +++---- src/core/algorithm/flat/flat_searcher.h | 6 ++-- src/core/algorithm/flat/flat_streamer.cc | 8 ++--- .../algorithm/flat/flat_streamer_dumper.h | 10 +++--- .../algorithm/flat/flat_streamer_entity.cc | 4 +-- .../algorithm/flat/flat_streamer_entity.h | 2 +- src/core/algorithm/flat/flat_utility.h | 35 +++++++------------ src/core/interface/indexes/flat_index.cc | 4 +-- src/core/utility/utility_params.h | 15 -------- .../algorithm/flat/flat_searcher_test.cpp | 8 ++--- .../core/algorithm/flat/flat_streamer_test.cc | 6 ++-- 12 files changed, 49 insertions(+), 73 deletions(-) diff --git a/src/core/algorithm/flat/flat_builder.cc b/src/core/algorithm/flat/flat_builder.cc index f4636d8..d5f3230 100644 --- a/src/core/algorithm/flat/flat_builder.cc +++ b/src/core/algorithm/flat/flat_builder.cc @@ -25,7 +25,7 @@ int FlatBuilder::init(const IndexMeta &meta, // Set the major order bool column_major_order = false; - if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order)) { + if (params.get(PARAM_FLAT_COLUMN_MAJOR_ORDER, &column_major_order)) { meta_.set_major_order(column_major_order ? IndexMeta::MO_COLUMN : IndexMeta::MO_ROW); } @@ -71,7 +71,7 @@ int FlatBuilder::init(const IndexMeta &meta, std::string tag = std::to_string(BATCH_SIZE); ailego::Params searcher_params; - searcher_params.set(FLAT_SEARCHER_BATCH_SIZE, BATCH_SIZE); + searcher_params.set(PARAM_FLAT_BATCH_SIZE, BATCH_SIZE); meta_.set_searcher("FlatSearcher" + tag, 0, searcher_params); meta_.set_builder("FlatBuilder" + tag, 0, params); return 0; @@ -151,7 +151,7 @@ int FlatBuilder::write_keys(const std::vector &keys, return IndexError_WriteData; } } - return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size, + return dumper->append(FLAT_SEGMENT_KEYS_SEG_ID, keys_size, keys_padding_size, 0); } @@ -179,7 +179,7 @@ int FlatBuilder::write_mapping(const std::vector &keys, return IndexError_WriteData; } } - return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size, + return dumper->append(FLAT_SEGMENT_MAPPING_SEG_ID, mapping_size, mapping_padding_size, 0); } @@ -236,7 +236,7 @@ int FlatBuilder::write_column_index(IndexDumper *dumper, return IndexError_WriteData; } } - return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size, + return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size, features_padding_size, 0); } @@ -271,7 +271,7 @@ int FlatBuilder::write_row_index(IndexDumper *dumper, return IndexError_WriteData; } } - return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size, + return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size, features_padding_size, 0); } diff --git a/src/core/algorithm/flat/flat_searcher.cc b/src/core/algorithm/flat/flat_searcher.cc index f9cdcfa..8c9b918 100644 --- a/src/core/algorithm/flat/flat_searcher.cc +++ b/src/core/algorithm/flat/flat_searcher.cc @@ -27,10 +27,10 @@ IndexProvider::Pointer FlatSearcher::create_provider(void) const { std::lock_guard lock(mapping_mutex_); if (mapping_.empty()) { - auto mapping_segment = container_->get(PARAM_FLAT_SEGMENT_MAPPING); + auto mapping_segment = container_->get(FLAT_SEGMENT_MAPPING_SEG_ID); if (!mapping_segment) { LOG_ERROR("Failed to fetch segment %s", - PARAM_FLAT_SEGMENT_MAPPING.c_str()); + FLAT_SEGMENT_MAPPING_SEG_ID.c_str()); return nullptr; } @@ -115,14 +115,14 @@ int FlatSearcher::load(IndexStorage::Pointer cntr, } } - auto keys_segment = cntr->get(PARAM_FLAT_SEGMENT_KEYS); + auto keys_segment = cntr->get(FLAT_SEGMENT_KEYS_SEG_ID); if (!keys_segment) { - LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str()); + LOG_ERROR("Failed to fetch segment %s", FLAT_SEGMENT_KEYS_SEG_ID.c_str()); return IndexError_NoExist; } - features_segment_ = cntr->get(PARAM_FLAT_SEGMENT_FEATURES); + features_segment_ = cntr->get(FLAT_SEGMENT_FEATURES_SEG_ID); if (!features_segment_) { - LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str()); + LOG_ERROR("Failed to fetch segment %s", FLAT_SEGMENT_KEYS_SEG_ID.c_str()); return IndexError_NoExist; } diff --git a/src/core/algorithm/flat/flat_searcher.h b/src/core/algorithm/flat/flat_searcher.h index e9a6151..12eccd6 100644 --- a/src/core/algorithm/flat/flat_searcher.h +++ b/src/core/algorithm/flat/flat_searcher.h @@ -32,8 +32,8 @@ class FlatSearcher : public IndexSearcher { //! Initialize Searcher int init(const ailego::Params &index_params) override { params_ = index_params; - read_block_size_ = FLAT_READ_BLOCK_SIZE; - index_params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_); + read_block_size_ = FLAT_DEFAULT_READ_BLOCK_SIZE; + index_params.get(PARAM_FLAT_READ_BLOCK_SIZE, &read_block_size_); return 0; } @@ -165,7 +165,7 @@ class FlatSearcher : public IndexSearcher { const uint64_t *keys_{nullptr}; std::map key_id_mapping_; uint32_t magic_{IndexContext::GenerateMagic()}; - uint32_t read_block_size_{FLAT_READ_BLOCK_SIZE}; + uint32_t read_block_size_{FLAT_DEFAULT_READ_BLOCK_SIZE}; bool column_major_order_{false}; IndexMeta meta_{}; IndexStorage::Pointer container_{}; diff --git a/src/core/algorithm/flat/flat_streamer.cc b/src/core/algorithm/flat/flat_streamer.cc index 4eb6383..20830b7 100644 --- a/src/core/algorithm/flat/flat_streamer.cc +++ b/src/core/algorithm/flat/flat_streamer.cc @@ -57,7 +57,7 @@ int FlatStreamer::init(const IndexMeta &imeta, } // 参数设置 - if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order_)) { + if (params.get(PARAM_FLAT_COLUMN_MAJOR_ORDER, &column_major_order_)) { meta_.set_major_order(column_major_order_ ? IndexMeta::MO_COLUMN : IndexMeta::MO_ROW); } @@ -99,9 +99,9 @@ int FlatStreamer::init(const IndexMeta &imeta, return IndexError_InvalidArgument; } - read_block_size_ = FLAT_READ_BLOCK_SIZE; - params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_); - params.get(FLAT_SEARCHER_USE_ID_MAP, &use_key_info_map_); + read_block_size_ = FLAT_DEFAULT_READ_BLOCK_SIZE; + params.get(PARAM_FLAT_READ_BLOCK_SIZE, &read_block_size_); + params.get(PARAM_FLAT_USE_ID_MAP, &use_key_info_map_); // entity init uint32_t block_vector_count = kDefaultBlockVecCount; diff --git a/src/core/algorithm/flat/flat_streamer_dumper.h b/src/core/algorithm/flat/flat_streamer_dumper.h index 7dd5c0d..7d3044b 100644 --- a/src/core/algorithm/flat/flat_streamer_dumper.h +++ b/src/core/algorithm/flat/flat_streamer_dumper.h @@ -140,7 +140,7 @@ class FlatStreamerDumper { } dump_size_ += padding.size(); } - return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size, + return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size, features_padding_size, 0); } @@ -177,7 +177,7 @@ class FlatStreamerDumper { } dump_size_ += padding.size(); } - return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size, + return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size, features_padding_size, 0); } @@ -200,8 +200,8 @@ class FlatStreamerDumper { } dump_size_ += padding.size(); } - return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size, - 0); + return dumper->append(FLAT_SEGMENT_KEYS_SEG_ID, keys_size, + keys_padding_size, 0); } int write_mapping(const std::vector &keys, IndexDumper *dumper) { @@ -230,7 +230,7 @@ class FlatStreamerDumper { } dump_size_ += padding.size(); } - return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size, + return dumper->append(FLAT_SEGMENT_MAPPING_SEG_ID, mapping_size, mapping_padding_size, 0); } diff --git a/src/core/algorithm/flat/flat_streamer_entity.cc b/src/core/algorithm/flat/flat_streamer_entity.cc index 80e26c4..639f383 100644 --- a/src/core/algorithm/flat/flat_streamer_entity.cc +++ b/src/core/algorithm/flat/flat_streamer_entity.cc @@ -688,7 +688,7 @@ int FlatStreamerEntity::load_storage(IndexStorage::Pointer storage) { size_t index_size = hd_segment->capacity(); for (size_t i = 1; i <= meta_.segment_count; ++i) { std::string segment_id = - ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i); + ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, i); auto seg = storage->get(segment_id); if (!seg || seg->data_size() < meta_.header.block_size) { LOG_ERROR("Failed to get segment %s, or invalid segment size", @@ -756,7 +756,7 @@ int FlatStreamerEntity::alloc_segment(void) { } std::string segment_id = - ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, index); + ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, index); size_t size = ailego_align(meta_.segment_size, ailego::MemoryHelper::PageSize()); auto segment = storage_->get(segment_id); diff --git a/src/core/algorithm/flat/flat_streamer_entity.h b/src/core/algorithm/flat/flat_streamer_entity.h index 0c4b535..277e8f2 100644 --- a/src/core/algorithm/flat/flat_streamer_entity.h +++ b/src/core/algorithm/flat/flat_streamer_entity.h @@ -219,7 +219,7 @@ class FlatStreamerEntity { const IndexStorage::Segment::Pointer get_segment(size_t index) const { for (size_t i = segments_.size(); i <= index; ++i) { auto segment_id = - ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i); + ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, i); auto segment = storage_->get(segment_id); if (!segment) { LOG_ERROR("Failed to get segment %s", segment_id.c_str()); diff --git a/src/core/algorithm/flat/flat_utility.h b/src/core/algorithm/flat/flat_utility.h index c84e741..3689ecc 100644 --- a/src/core/algorithm/flat/flat_utility.h +++ b/src/core/algorithm/flat/flat_utility.h @@ -25,32 +25,21 @@ namespace zvec { namespace core { //! The default size of reading a block -static constexpr uint32_t FLAT_READ_BLOCK_SIZE = 4 * 1024 * 1024; -static const std::string FLAT_LINEAR_META_SEG_ID = "linear.linear_meta"; -static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID = - "linear.linear_list_head"; +static constexpr uint32_t FLAT_DEFAULT_READ_BLOCK_SIZE = 4 * 1024 * 1024; +static const std::string FLAT_LINEAR_META_SEG_ID = "flat.linear_meta"; +static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID = "flat.linear_list_head"; -// reducer params -static const std::string PARAM_FLAT_REDUCER_WORKING_PATH( - "proxima.linear.reducer.working_path"); -static const std::string PARAM_FLAT_REDUCER_NUM_OF_ADD_THREADS( - "proxima.linear.reducer.num_of_add_threads"); -static const std::string PARAM_FLAT_REDUCER_INDEX_NAME( - "proxima.linear.reducer.index_name"); +static const std::string FLAT_SEGMENT_KEYS_SEG_ID("flat.keys"); +static const std::string FLAT_SEGMENT_FEATURES_SEG_ID("flat.features"); +static const std::string FLAT_SEGMENT_MAPPING_SEG_ID("flat.mapping"); // index params -static const std::string PARAM_FLAT_SEGMENT_KEYS("linear.keys"); -static const std::string PARAM_FLAT_SEGMENT_FEATURES("linear.features"); -static const std::string PARAM_FLAT_SEGMENT_MAPPING("linear.mapping"); - -static const std::string FLAT_BUILDER_COLUMN_MAJOR_ORDER( - "proxima.linear.builder.column_major_order"); -static const std::string FLAT_SEARCHER_BATCH_SIZE( - "proxima.linear.searcher.batch_size"); -static const std::string FLAT_SEARCHER_READ_BLOCK_SIZE( - "proxima.linear.searcher.read_block_size"); -static const std::string FLAT_SEARCHER_USE_ID_MAP( - "proxima.linear.searcher.use_id_map"); +static const std::string PARAM_FLAT_COLUMN_MAJOR_ORDER( + "proxima.flat.column_major_order"); +static const std::string PARAM_FLAT_BATCH_SIZE("proxima.flat.batch_size"); +static const std::string PARAM_FLAT_READ_BLOCK_SIZE( + "proxima.flat.read_block_size"); +static const std::string PARAM_FLAT_USE_ID_MAP("proxima.flat.use_id_map"); //! Determines if a number is equal to two to the power of n. template diff --git a/src/core/interface/indexes/flat_index.cc b/src/core/interface/indexes/flat_index.cc index ec196bf..0e7e5fd 100644 --- a/src/core/interface/indexes/flat_index.cc +++ b/src/core/interface/indexes/flat_index.cc @@ -25,9 +25,9 @@ namespace zvec::core_interface { int FlatIndex::CreateAndInitStreamer(const BaseIndexParam ¶m) { param_ = dynamic_cast(param); - proxima_index_params_.set(core::FLAT_BUILDER_COLUMN_MAJOR_ORDER, + proxima_index_params_.set(core::PARAM_FLAT_COLUMN_MAJOR_ORDER, param_.major_order == IndexMeta::MO_COLUMN); - proxima_index_params_.set(core::FLAT_SEARCHER_USE_ID_MAP, param_.use_id_map); + proxima_index_params_.set(core::PARAM_FLAT_USE_ID_MAP, param_.use_id_map); if (is_sparse_) { streamer_ = core::IndexFactory::CreateStreamer("FlatSparseStreamer"); } else { diff --git a/src/core/utility/utility_params.h b/src/core/utility/utility_params.h index dee70cc..f4f8dd1 100644 --- a/src/core/utility/utility_params.h +++ b/src/core/utility/utility_params.h @@ -21,21 +21,6 @@ namespace core { static const std::string INDEX_META_SEGMENT_NAME = "IndexMeta"; static const std::string INDEX_VERSION_SEGMENT_NAME = "IndexVersion"; -//! BruteForceBuilder/BruteForceSearcher -static const std::string BRUTE_FORCE_SEGMENT_KEYS = "linear.keys"; -static const std::string BRUTE_FORCE_SEGMENT_FEATURES = "linear.features"; -static const std::string BRUTE_FORCE_SEGMENT_MAPPING = "linear.mapping"; - -//! BruteForceBuilder -static const std::string BRUTE_FORCE_BUILDER_COLUMN_MAJOR_ORDER = - "proxima.linear.builder.column_major_order"; - -//! BruteForceSearcher -static const std::string BRUTE_FORCE_SEARCHER_BATCH_SIZE = - "proxima.linear.searcher.batch_size"; -static const std::string BRUTE_FORCE_SEARCHER_READ_BLOCK_SIZE = - "proxima.linear.searcher.read_block_size"; - //! FileLogger static const std::string FILE_LOGGER_PATH = "proxima.file.logger.path"; diff --git a/tests/core/algorithm/flat/flat_searcher_test.cpp b/tests/core/algorithm/flat/flat_searcher_test.cpp index 21c1783..dc79211 100644 --- a/tests/core/algorithm/flat/flat_searcher_test.cpp +++ b/tests/core/algorithm/flat/flat_searcher_test.cpp @@ -883,7 +883,7 @@ TEST(FlatSearcher, ColumnBatch_Binary32) { meta1.set_major_order(IndexMeta::MO_ROW); Params params1; - params1.set("proxima.linear.builder.column_major_order", true); + params1.set(PARAM_FLAT_COLUMN_MAJOR_ORDER, true); BuildIndex(meta1, params1, holder, INDEX_PATH + ".1"); IndexMeta meta2; @@ -984,7 +984,7 @@ TEST(FlatSearcher, ColumnBatch_FP32) { // column_major_order in params Params params1; - params1.set("proxima.linear.builder.column_major_order", + params1.set(PARAM_FLAT_COLUMN_MAJOR_ORDER, true); // make it MO_COLUMN BuildIndex(meta1, params1, holder, INDEX_PATH + ".1"); @@ -1090,7 +1090,7 @@ TEST(FlatSearcher, ColumnBatch_FP16) { // column_major_order in params Params params1; - params1.set("proxima.linear.builder.column_major_order", + params1.set(PARAM_FLAT_COLUMN_MAJOR_ORDER, true); // make it MO_COLUMN BuildIndex(meta1, params1, holder, INDEX_PATH + ".1"); @@ -1196,7 +1196,7 @@ TEST(FlatSearcher, ColumnBatch_INT8) { // column_major_order in params Params params1; - params1.set("proxima.linear.builder.column_major_order", + params1.set(PARAM_FLAT_COLUMN_MAJOR_ORDER, true); // make it MO_COLUMN BuildIndex(meta1, params1, holder, INDEX_PATH + ".1"); diff --git a/tests/core/algorithm/flat/flat_streamer_test.cc b/tests/core/algorithm/flat/flat_streamer_test.cc index 15ef4a4..68f3dbc 100644 --- a/tests/core/algorithm/flat/flat_streamer_test.cc +++ b/tests/core/algorithm/flat/flat_streamer_test.cc @@ -22,6 +22,8 @@ #include #include #include +#include "ailego/encoding/json/mod_json.h" +#include "algorithm/flat/flat_utility.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -353,7 +355,7 @@ TEST_F(FlatStreamerTest, TestOpenClose) { IndexMeta meta(IndexMeta::DataType::DT_FP32, dim); meta.set_metric("SquaredEuclidean", 0, Params()); Params params; - // params.set("proxima.linear.builder.column_major_order", false); + // params.set(PARAM_FLAT_COLUMN_MAJOR_ORDER, false); auto storage1 = IndexFactory::CreateStorage("MMapFileStorage"); auto storage2 = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage1); @@ -1151,7 +1153,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID2) { IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatStreamer"); Params read_params; - read_params.set("proxima.linear.searcher.use_id_map", false); + read_params.set(PARAM_FLAT_USE_ID_MAP, false); Params read_stg_params; auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, read_storage->init(read_stg_params));