refactor: flat param string cleanup (#28)
* refactor: flat param string cleanup * fix
This commit is contained in:
@@ -25,7 +25,7 @@ int FlatBuilder<BATCH_SIZE>::init(const IndexMeta &meta,
|
||||
|
||||
// Set the major order
|
||||
bool column_major_order = false;
|
||||
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order)) {
|
||||
if (params.get(PARAM_FLAT_COLUMN_MAJOR_ORDER, &column_major_order)) {
|
||||
meta_.set_major_order(column_major_order ? IndexMeta::MO_COLUMN
|
||||
: IndexMeta::MO_ROW);
|
||||
}
|
||||
@@ -71,7 +71,7 @@ int FlatBuilder<BATCH_SIZE>::init(const IndexMeta &meta,
|
||||
|
||||
std::string tag = std::to_string(BATCH_SIZE);
|
||||
ailego::Params searcher_params;
|
||||
searcher_params.set(FLAT_SEARCHER_BATCH_SIZE, BATCH_SIZE);
|
||||
searcher_params.set(PARAM_FLAT_BATCH_SIZE, BATCH_SIZE);
|
||||
meta_.set_searcher("FlatSearcher" + tag, 0, searcher_params);
|
||||
meta_.set_builder("FlatBuilder" + tag, 0, params);
|
||||
return 0;
|
||||
@@ -151,7 +151,7 @@ int FlatBuilder<BATCH_SIZE>::write_keys(const std::vector<uint64_t> &keys,
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
|
||||
return dumper->append(FLAT_SEGMENT_KEYS_SEG_ID, keys_size, keys_padding_size,
|
||||
0);
|
||||
}
|
||||
|
||||
@@ -179,7 +179,7 @@ int FlatBuilder<BATCH_SIZE>::write_mapping(const std::vector<uint64_t> &keys,
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
|
||||
return dumper->append(FLAT_SEGMENT_MAPPING_SEG_ID, mapping_size,
|
||||
mapping_padding_size, 0);
|
||||
}
|
||||
|
||||
@@ -236,7 +236,7 @@ int FlatBuilder<BATCH_SIZE>::write_column_index(IndexDumper *dumper,
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
|
||||
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
|
||||
features_padding_size, 0);
|
||||
}
|
||||
|
||||
@@ -271,7 +271,7 @@ int FlatBuilder<BATCH_SIZE>::write_row_index(IndexDumper *dumper,
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
|
||||
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
|
||||
features_padding_size, 0);
|
||||
}
|
||||
|
||||
|
||||
@@ -27,10 +27,10 @@ IndexProvider::Pointer FlatSearcher<BATCH_SIZE>::create_provider(void) const {
|
||||
std::lock_guard<std::mutex> lock(mapping_mutex_);
|
||||
|
||||
if (mapping_.empty()) {
|
||||
auto mapping_segment = container_->get(PARAM_FLAT_SEGMENT_MAPPING);
|
||||
auto mapping_segment = container_->get(FLAT_SEGMENT_MAPPING_SEG_ID);
|
||||
if (!mapping_segment) {
|
||||
LOG_ERROR("Failed to fetch segment %s",
|
||||
PARAM_FLAT_SEGMENT_MAPPING.c_str());
|
||||
FLAT_SEGMENT_MAPPING_SEG_ID.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@@ -115,14 +115,14 @@ int FlatSearcher<BATCH_SIZE>::load(IndexStorage::Pointer cntr,
|
||||
}
|
||||
}
|
||||
|
||||
auto keys_segment = cntr->get(PARAM_FLAT_SEGMENT_KEYS);
|
||||
auto keys_segment = cntr->get(FLAT_SEGMENT_KEYS_SEG_ID);
|
||||
if (!keys_segment) {
|
||||
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
|
||||
LOG_ERROR("Failed to fetch segment %s", FLAT_SEGMENT_KEYS_SEG_ID.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
features_segment_ = cntr->get(PARAM_FLAT_SEGMENT_FEATURES);
|
||||
features_segment_ = cntr->get(FLAT_SEGMENT_FEATURES_SEG_ID);
|
||||
if (!features_segment_) {
|
||||
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
|
||||
LOG_ERROR("Failed to fetch segment %s", FLAT_SEGMENT_KEYS_SEG_ID.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
|
||||
@@ -32,8 +32,8 @@ class FlatSearcher : public IndexSearcher {
|
||||
//! Initialize Searcher
|
||||
int init(const ailego::Params &index_params) override {
|
||||
params_ = index_params;
|
||||
read_block_size_ = FLAT_READ_BLOCK_SIZE;
|
||||
index_params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
|
||||
read_block_size_ = FLAT_DEFAULT_READ_BLOCK_SIZE;
|
||||
index_params.get(PARAM_FLAT_READ_BLOCK_SIZE, &read_block_size_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -165,7 +165,7 @@ class FlatSearcher : public IndexSearcher {
|
||||
const uint64_t *keys_{nullptr};
|
||||
std::map<key_t, node_id_t> key_id_mapping_;
|
||||
uint32_t magic_{IndexContext::GenerateMagic()};
|
||||
uint32_t read_block_size_{FLAT_READ_BLOCK_SIZE};
|
||||
uint32_t read_block_size_{FLAT_DEFAULT_READ_BLOCK_SIZE};
|
||||
bool column_major_order_{false};
|
||||
IndexMeta meta_{};
|
||||
IndexStorage::Pointer container_{};
|
||||
|
||||
@@ -57,7 +57,7 @@ int FlatStreamer<BATCH_SIZE>::init(const IndexMeta &imeta,
|
||||
}
|
||||
|
||||
// 参数设置
|
||||
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order_)) {
|
||||
if (params.get(PARAM_FLAT_COLUMN_MAJOR_ORDER, &column_major_order_)) {
|
||||
meta_.set_major_order(column_major_order_ ? IndexMeta::MO_COLUMN
|
||||
: IndexMeta::MO_ROW);
|
||||
}
|
||||
@@ -99,9 +99,9 @@ int FlatStreamer<BATCH_SIZE>::init(const IndexMeta &imeta,
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
read_block_size_ = FLAT_READ_BLOCK_SIZE;
|
||||
params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
|
||||
params.get(FLAT_SEARCHER_USE_ID_MAP, &use_key_info_map_);
|
||||
read_block_size_ = FLAT_DEFAULT_READ_BLOCK_SIZE;
|
||||
params.get(PARAM_FLAT_READ_BLOCK_SIZE, &read_block_size_);
|
||||
params.get(PARAM_FLAT_USE_ID_MAP, &use_key_info_map_);
|
||||
|
||||
// entity init
|
||||
uint32_t block_vector_count = kDefaultBlockVecCount;
|
||||
|
||||
@@ -140,7 +140,7 @@ class FlatStreamerDumper {
|
||||
}
|
||||
dump_size_ += padding.size();
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
|
||||
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
|
||||
features_padding_size, 0);
|
||||
}
|
||||
|
||||
@@ -177,7 +177,7 @@ class FlatStreamerDumper {
|
||||
}
|
||||
dump_size_ += padding.size();
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
|
||||
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
|
||||
features_padding_size, 0);
|
||||
}
|
||||
|
||||
@@ -200,8 +200,8 @@ class FlatStreamerDumper {
|
||||
}
|
||||
dump_size_ += padding.size();
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
|
||||
0);
|
||||
return dumper->append(FLAT_SEGMENT_KEYS_SEG_ID, keys_size,
|
||||
keys_padding_size, 0);
|
||||
}
|
||||
|
||||
int write_mapping(const std::vector<uint64_t> &keys, IndexDumper *dumper) {
|
||||
@@ -230,7 +230,7 @@ class FlatStreamerDumper {
|
||||
}
|
||||
dump_size_ += padding.size();
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
|
||||
return dumper->append(FLAT_SEGMENT_MAPPING_SEG_ID, mapping_size,
|
||||
mapping_padding_size, 0);
|
||||
}
|
||||
|
||||
|
||||
@@ -688,7 +688,7 @@ int FlatStreamerEntity::load_storage(IndexStorage::Pointer storage) {
|
||||
size_t index_size = hd_segment->capacity();
|
||||
for (size_t i = 1; i <= meta_.segment_count; ++i) {
|
||||
std::string segment_id =
|
||||
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
|
||||
ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, i);
|
||||
auto seg = storage->get(segment_id);
|
||||
if (!seg || seg->data_size() < meta_.header.block_size) {
|
||||
LOG_ERROR("Failed to get segment %s, or invalid segment size",
|
||||
@@ -756,7 +756,7 @@ int FlatStreamerEntity::alloc_segment(void) {
|
||||
}
|
||||
|
||||
std::string segment_id =
|
||||
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, index);
|
||||
ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, index);
|
||||
size_t size =
|
||||
ailego_align(meta_.segment_size, ailego::MemoryHelper::PageSize());
|
||||
auto segment = storage_->get(segment_id);
|
||||
|
||||
@@ -219,7 +219,7 @@ class FlatStreamerEntity {
|
||||
const IndexStorage::Segment::Pointer get_segment(size_t index) const {
|
||||
for (size_t i = segments_.size(); i <= index; ++i) {
|
||||
auto segment_id =
|
||||
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
|
||||
ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, i);
|
||||
auto segment = storage_->get(segment_id);
|
||||
if (!segment) {
|
||||
LOG_ERROR("Failed to get segment %s", segment_id.c_str());
|
||||
|
||||
@@ -25,32 +25,21 @@ namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! The default size of reading a block
|
||||
static constexpr uint32_t FLAT_READ_BLOCK_SIZE = 4 * 1024 * 1024;
|
||||
static const std::string FLAT_LINEAR_META_SEG_ID = "linear.linear_meta";
|
||||
static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID =
|
||||
"linear.linear_list_head";
|
||||
static constexpr uint32_t FLAT_DEFAULT_READ_BLOCK_SIZE = 4 * 1024 * 1024;
|
||||
static const std::string FLAT_LINEAR_META_SEG_ID = "flat.linear_meta";
|
||||
static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID = "flat.linear_list_head";
|
||||
|
||||
// reducer params
|
||||
static const std::string PARAM_FLAT_REDUCER_WORKING_PATH(
|
||||
"proxima.linear.reducer.working_path");
|
||||
static const std::string PARAM_FLAT_REDUCER_NUM_OF_ADD_THREADS(
|
||||
"proxima.linear.reducer.num_of_add_threads");
|
||||
static const std::string PARAM_FLAT_REDUCER_INDEX_NAME(
|
||||
"proxima.linear.reducer.index_name");
|
||||
static const std::string FLAT_SEGMENT_KEYS_SEG_ID("flat.keys");
|
||||
static const std::string FLAT_SEGMENT_FEATURES_SEG_ID("flat.features");
|
||||
static const std::string FLAT_SEGMENT_MAPPING_SEG_ID("flat.mapping");
|
||||
|
||||
// index params
|
||||
static const std::string PARAM_FLAT_SEGMENT_KEYS("linear.keys");
|
||||
static const std::string PARAM_FLAT_SEGMENT_FEATURES("linear.features");
|
||||
static const std::string PARAM_FLAT_SEGMENT_MAPPING("linear.mapping");
|
||||
|
||||
static const std::string FLAT_BUILDER_COLUMN_MAJOR_ORDER(
|
||||
"proxima.linear.builder.column_major_order");
|
||||
static const std::string FLAT_SEARCHER_BATCH_SIZE(
|
||||
"proxima.linear.searcher.batch_size");
|
||||
static const std::string FLAT_SEARCHER_READ_BLOCK_SIZE(
|
||||
"proxima.linear.searcher.read_block_size");
|
||||
static const std::string FLAT_SEARCHER_USE_ID_MAP(
|
||||
"proxima.linear.searcher.use_id_map");
|
||||
static const std::string PARAM_FLAT_COLUMN_MAJOR_ORDER(
|
||||
"proxima.flat.column_major_order");
|
||||
static const std::string PARAM_FLAT_BATCH_SIZE("proxima.flat.batch_size");
|
||||
static const std::string PARAM_FLAT_READ_BLOCK_SIZE(
|
||||
"proxima.flat.read_block_size");
|
||||
static const std::string PARAM_FLAT_USE_ID_MAP("proxima.flat.use_id_map");
|
||||
|
||||
//! Determines if a number is equal to two to the power of n.
|
||||
template <size_t K>
|
||||
|
||||
Reference in New Issue
Block a user