Initial commit
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
#message(STATUS "PROJECT_ROOT_DIR = ${PROJECT_ROOT_DIR}")
|
||||
cc_library(
|
||||
NAME core_knn_flat
|
||||
STATIC SHARED STRICT ALWAYS_LINK
|
||||
SRCS *.cc
|
||||
LIBS core_framework
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm ${PROJECT_ROOT_DIR}/src/core/framework
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
@@ -0,0 +1,284 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "flat_builder.h"
|
||||
#include "flat_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatBuilder<BATCH_SIZE>::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
meta_ = meta;
|
||||
|
||||
// Set the major order
|
||||
bool column_major_order = false;
|
||||
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order)) {
|
||||
meta_.set_major_order(column_major_order ? IndexMeta::MO_COLUMN
|
||||
: IndexMeta::MO_ROW);
|
||||
}
|
||||
|
||||
// Verify column major order
|
||||
if (meta_.major_order() != IndexMeta::MO_ROW) {
|
||||
IndexMeta::DataType dt = meta_.data_type();
|
||||
|
||||
bool support_column_major = false;
|
||||
if ((dt != IndexMeta::DataType::DT_FP32 &&
|
||||
dt != IndexMeta::DataType::DT_FP16 &&
|
||||
dt != IndexMeta::DataType::DT_INT8 && dt != IndexMeta::DT_INT4 &&
|
||||
dt != IndexMeta::DT_BINARY32 && dt != IndexMeta::DT_BINARY64) ||
|
||||
(meta_.unit_size() != IndexMeta::UnitSizeof(dt))) {
|
||||
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
|
||||
LOG_ERROR("Unsupported type %d with unit size %u.", dt,
|
||||
meta_.unit_size());
|
||||
return IndexError_Unsupported;
|
||||
} else {
|
||||
support_column_major = false;
|
||||
}
|
||||
}
|
||||
if (meta_.element_size() % IndexMeta::AlignSizeof(dt) != 0) {
|
||||
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
|
||||
LOG_ERROR("Unsupported type %d with dimension %u.", dt,
|
||||
meta_.dimension());
|
||||
return IndexError_Unsupported;
|
||||
} else {
|
||||
support_column_major = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (meta_.major_order() == IndexMeta::MO_UNDEFINED &&
|
||||
support_column_major) {
|
||||
meta_.set_major_order(IndexMeta::MO_COLUMN);
|
||||
}
|
||||
}
|
||||
|
||||
if (!VerifyMetric(meta_)) {
|
||||
LOG_ERROR("Invalid index measure %s.", meta_.metric_name().c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::string tag = std::to_string(BATCH_SIZE);
|
||||
ailego::Params searcher_params;
|
||||
searcher_params.set(FLAT_SEARCHER_BATCH_SIZE, BATCH_SIZE);
|
||||
meta_.set_searcher("FlatSearcher" + tag, 0, searcher_params);
|
||||
meta_.set_builder("FlatBuilder" + tag, 0, params);
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatBuilder<BATCH_SIZE>::build(IndexThreads::Pointer,
|
||||
IndexHolder::Pointer holder) {
|
||||
ailego::ElapsedTime stamp;
|
||||
if (!holder->is_matched(meta_)) {
|
||||
LOG_ERROR("The holder is unmatched with initialized meta.");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
holder_ = std::move(holder);
|
||||
stats_.set_built_count(holder_->count());
|
||||
stats_.set_built_costtime(stamp.milli_seconds());
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatBuilder<BATCH_SIZE>::dump(const IndexDumper::Pointer &dumper) {
|
||||
ailego::ElapsedTime stamp;
|
||||
if (!holder_) {
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
std::vector<uint64_t> keys;
|
||||
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
|
||||
int error_code = this->write_column_index(dumper.get(), &keys);
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
} else {
|
||||
int error_code = this->write_row_index(dumper.get(), &keys);
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
}
|
||||
|
||||
int error_code = this->write_keys(keys, dumper.get());
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
|
||||
error_code = this->write_mapping(keys, dumper.get());
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
|
||||
error_code = IndexHelper::SerializeToDumper(meta_, dumper.get());
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
|
||||
holder_ = nullptr;
|
||||
stats_.set_dumped_count(keys.size());
|
||||
stats_.set_dumped_costtime(stamp.milli_seconds());
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatBuilder<BATCH_SIZE>::write_keys(const std::vector<uint64_t> &keys,
|
||||
IndexDumper *dumper) {
|
||||
size_t keys_size = keys.size() * sizeof(uint64_t);
|
||||
size_t keys_padding_size = ailego_align(keys_size, 32) - keys_size;
|
||||
if (dumper->write(keys.data(), keys_size) != keys_size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
// Write the padding if need
|
||||
if (keys_padding_size) {
|
||||
std::string padding(keys_padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
|
||||
0);
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatBuilder<BATCH_SIZE>::write_mapping(const std::vector<uint64_t> &keys,
|
||||
IndexDumper *dumper) {
|
||||
std::vector<uint32_t> mapping(keys.size());
|
||||
std::iota(mapping.begin(), mapping.end(), 0);
|
||||
std::sort(
|
||||
mapping.begin(), mapping.end(),
|
||||
[&keys](uint32_t lhs, uint32_t rhs) { return (keys[lhs] < keys[rhs]); });
|
||||
|
||||
size_t mapping_size = mapping.size() * sizeof(uint32_t);
|
||||
size_t mapping_padding_size = ailego_align(mapping_size, 32) - mapping_size;
|
||||
if (dumper->write(mapping.data(), mapping_size) != mapping_size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
// Write the padding if need
|
||||
if (mapping_padding_size) {
|
||||
std::string padding(mapping_padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
|
||||
mapping_padding_size, 0);
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
template <typename T>
|
||||
int FlatBuilder<BATCH_SIZE>::write_column_index(IndexDumper *dumper,
|
||||
std::vector<uint64_t> *keys) {
|
||||
auto iter = holder_->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Failed to create iterator of holder");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
// Write features
|
||||
size_t element_size = holder_->element_size();
|
||||
size_t block_size = element_size * BATCH_SIZE;
|
||||
std::string block1, block2;
|
||||
block1.reserve(block_size);
|
||||
block2.reserve(block_size);
|
||||
|
||||
for (; iter->is_valid(); iter->next()) {
|
||||
block1.append(reinterpret_cast<const char *>(iter->data()), element_size);
|
||||
keys->emplace_back(iter->key());
|
||||
|
||||
if (block1.size() == block_size) {
|
||||
ailego::MatrixHelper::Transpose<T, BATCH_SIZE>(
|
||||
block1.data(), element_size / sizeof(T), (void *)block2.data());
|
||||
|
||||
if (dumper->write(block2.data(), block_size) != block_size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s",
|
||||
dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
block1.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (!block1.empty()) {
|
||||
if (dumper->write(block1.data(), block1.size()) != block1.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
|
||||
// Write the padding if need
|
||||
size_t features_size = keys->size() * element_size;
|
||||
size_t features_padding_size =
|
||||
ailego_align(features_size, 32) - features_size;
|
||||
if (features_padding_size) {
|
||||
std::string padding(features_padding_size, '\0');
|
||||
|
||||
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
|
||||
features_padding_size, 0);
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatBuilder<BATCH_SIZE>::write_row_index(IndexDumper *dumper,
|
||||
std::vector<uint64_t> *keys) {
|
||||
auto iter = holder_->create_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Failed to create iterator of holder");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
// Write features
|
||||
size_t element_size = holder_->element_size();
|
||||
for (; iter->is_valid(); iter->next()) {
|
||||
if (dumper->write(iter->data(), element_size) != element_size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
keys->emplace_back(iter->key());
|
||||
}
|
||||
|
||||
// Write the padding if need
|
||||
size_t features_size = keys->size() * element_size;
|
||||
size_t features_padding_size =
|
||||
ailego_align(features_size, 32) - features_size;
|
||||
if (features_padding_size) {
|
||||
std::string padding(features_padding_size, '\0');
|
||||
|
||||
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
|
||||
features_padding_size, 0);
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_BUILDER_ALIAS(LinearBuilder, FlatBuilder<32>);
|
||||
INDEX_FACTORY_REGISTER_BUILDER_ALIAS(FlatBuilder, FlatBuilder<32>);
|
||||
INDEX_FACTORY_REGISTER_BUILDER_ALIAS(FlatBuilder16, FlatBuilder<16>);
|
||||
INDEX_FACTORY_REGISTER_BUILDER_ALIAS(FlatBuilder32, FlatBuilder<32>);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <framework/index_builder.h>
|
||||
#include <framework/index_helper.h>
|
||||
#include "flat_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Flat Builder
|
||||
*/
|
||||
template <size_t BATCH_SIZE>
|
||||
class FlatBuilder : public IndexBuilder {
|
||||
public:
|
||||
//! Destructor
|
||||
virtual ~FlatBuilder(void) {}
|
||||
|
||||
//! Initialize the builder
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup the builder
|
||||
int cleanup(void) override {
|
||||
holder_ = nullptr;
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Train the data
|
||||
int train(IndexThreads::Pointer, IndexHolder::Pointer) override {
|
||||
stats_.set_trained_count(0u);
|
||||
stats_.set_trained_costtime(0u);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Train the data
|
||||
int train(const IndexTrainer::Pointer &) override {
|
||||
stats_.set_trained_count(0u);
|
||||
stats_.set_trained_costtime(0u);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Build the index
|
||||
int build(IndexThreads::Pointer, IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Dump index into storage
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
const IndexBuilder::Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Dump index keys
|
||||
int write_keys(const std::vector<uint64_t> &keys, IndexDumper *dumper);
|
||||
|
||||
//! Dump index keys mapping
|
||||
int write_mapping(const std::vector<uint64_t> &keys, IndexDumper *dumper);
|
||||
|
||||
//! Dump index using column-major-order format
|
||||
template <typename T>
|
||||
int write_column_index(IndexDumper *dumper, std::vector<uint64_t> *keys);
|
||||
|
||||
//! Dump index using column-major-order format
|
||||
int write_column_index(IndexDumper *dumper, std::vector<uint64_t> *keys) {
|
||||
switch (IndexMeta::AlignSizeof(meta_.data_type())) {
|
||||
case 2:
|
||||
return this->write_column_index<uint16_t>(dumper, keys);
|
||||
case 4:
|
||||
return this->write_column_index<uint32_t>(dumper, keys);
|
||||
case 8:
|
||||
return this->write_column_index<uint64_t>(dumper, keys);
|
||||
default:
|
||||
ailego_check_with(0, "BAD CASE");
|
||||
}
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
//! Dump index using row-major-order format
|
||||
int write_row_index(IndexDumper *dumper, std::vector<uint64_t> *keys);
|
||||
|
||||
private:
|
||||
IndexMeta meta_{};
|
||||
IndexBuilder::Stats stats_{};
|
||||
IndexHolder::Pointer holder_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,184 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
#include "flat_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Brute Force Distance Tuple
|
||||
*/
|
||||
template <size_t K, typename = void>
|
||||
class FlatDistanceTuple;
|
||||
|
||||
/*! Brute Force Distance Tuple
|
||||
*/
|
||||
template <>
|
||||
class FlatDistanceTuple<1> {
|
||||
public:
|
||||
//! Retrieve non-zero if all distances are valid.
|
||||
bool is_valid(void) const {
|
||||
return !!distance_;
|
||||
}
|
||||
|
||||
//! Retrieve non-zero if a distance is valid.
|
||||
bool is_valid(size_t m) const {
|
||||
return m == 1 && !!distance_;
|
||||
}
|
||||
|
||||
//! Initialize the distance tuple
|
||||
void initialize(const IndexMetric &measure) {
|
||||
distance_ = measure.distance_matrix(1, 1);
|
||||
}
|
||||
|
||||
//! Initialize the distance tuple
|
||||
void initialize(const IndexMetric &measure, size_t m) {
|
||||
distance_ = measure.distance_matrix(m, 1);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
template <size_t M>
|
||||
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
|
||||
typename std::enable_if<M == 1>::type {
|
||||
distance_(m, q, dim, out);
|
||||
}
|
||||
|
||||
private:
|
||||
IndexMetric::MatrixDistance distance_{};
|
||||
};
|
||||
|
||||
/*! Brute Force Distance Tuple
|
||||
*/
|
||||
template <size_t K>
|
||||
class FlatDistanceTuple<
|
||||
K, typename std::enable_if<IsEqualPowerofTwo<K>::value>::type> {
|
||||
public:
|
||||
//! Retrieve non-zero if all distances are valid.
|
||||
bool is_valid(void) const {
|
||||
return (distance_tuple_.is_valid() && !!distance_);
|
||||
}
|
||||
|
||||
//! Retrieve non-zero if a distance is valid.
|
||||
bool is_valid(size_t m) const {
|
||||
return (m == K ? (!!distance_)
|
||||
: (m < K ? distance_tuple_.is_valid(m) : false));
|
||||
}
|
||||
|
||||
//! Initialize the distance tuple
|
||||
void initialize(const IndexMetric &measure) {
|
||||
distance_tuple_.initialize(measure);
|
||||
distance_ = measure.distance_matrix(K, 1);
|
||||
}
|
||||
|
||||
//! Initialize the distance tuple
|
||||
void initialize(const IndexMetric &measure, size_t m) {
|
||||
distance_tuple_.initialize(measure, m);
|
||||
distance_ = measure.distance_matrix(m, K);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
template <size_t M>
|
||||
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
|
||||
typename std::enable_if<K == M>::type {
|
||||
distance_(m, q, dim, out);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
template <size_t M>
|
||||
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
|
||||
typename std::enable_if<(K > M) && IsEqualPowerofTwo<M>::value>::type {
|
||||
distance_tuple_.template distance<M>(m, q, dim, out);
|
||||
}
|
||||
|
||||
private:
|
||||
FlatDistanceTuple<(K >> 1)> distance_tuple_{};
|
||||
IndexMetric::MatrixDistance distance_{};
|
||||
};
|
||||
|
||||
/*! Brute Force Distance Matrix
|
||||
*/
|
||||
template <size_t K, typename = void>
|
||||
class FlatDistanceMatrix;
|
||||
|
||||
/*! Brute Force Distance Matrix
|
||||
*/
|
||||
template <>
|
||||
class FlatDistanceMatrix<1> {
|
||||
public:
|
||||
//! Retrieve non-zero if all distances are valid.
|
||||
bool is_valid(void) const {
|
||||
return (!!distance_);
|
||||
}
|
||||
|
||||
//! Initialize the distance tuple
|
||||
void initialize(const IndexMetric &measure) {
|
||||
distance_ = measure.distance_matrix(1, 1);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
template <size_t M, size_t N = 1u>
|
||||
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
|
||||
typename std::enable_if<M == 1u && N == 1u>::type {
|
||||
distance_(m, q, dim, out);
|
||||
}
|
||||
|
||||
private:
|
||||
IndexMetric::MatrixDistance distance_{};
|
||||
};
|
||||
|
||||
/*! Brute Force Distance Matrix
|
||||
*/
|
||||
template <size_t K>
|
||||
class FlatDistanceMatrix<
|
||||
K, typename std::enable_if<IsEqualPowerofTwo<K>::value>::type> {
|
||||
public:
|
||||
//! Retrieve non-zero if all distances are valid.
|
||||
bool is_valid(void) const {
|
||||
return (tuple_h_.is_valid() && tuple_v_.is_valid());
|
||||
}
|
||||
|
||||
//! Retrieve non-zero if a distance is valid.
|
||||
bool is_valid(size_t m, size_t n) const {
|
||||
return (m == K ? tuple_h_.is_valid(n)
|
||||
: (m < K && n == 1 ? tuple_v_.is_valid(m) : false));
|
||||
}
|
||||
|
||||
//! Initialize the distance tuple
|
||||
void initialize(const IndexMetric &measure) {
|
||||
tuple_h_.initialize(measure, K);
|
||||
tuple_v_.initialize(measure);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
template <size_t M, size_t N>
|
||||
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
|
||||
typename std::enable_if<(K == M) && (K >= N)>::type {
|
||||
tuple_h_.template distance<N>(m, q, dim, out);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
template <size_t M, size_t N = 1u>
|
||||
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
|
||||
typename std::enable_if<(K > M) && (N == 1u)>::type {
|
||||
tuple_v_.template distance<M>(m, q, dim, out);
|
||||
}
|
||||
|
||||
private:
|
||||
FlatDistanceTuple<K> tuple_h_{};
|
||||
FlatDistanceTuple<(K >> 1)> tuple_v_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,184 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/container/bitmap.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
using node_id_t = uint32_t;
|
||||
using key_t = uint64_t;
|
||||
using level_t = int32_t;
|
||||
using dist_t = float;
|
||||
using TopkHeap = ailego::KeyValueHeap<node_id_t, dist_t>;
|
||||
using CandidateHeap =
|
||||
ailego::KeyValueHeap<node_id_t, dist_t, std::greater<dist_t>>;
|
||||
constexpr node_id_t kInvalidNodeId = static_cast<node_id_t>(-1);
|
||||
constexpr key_t kInvalidKey = static_cast<key_t>(-1);
|
||||
|
||||
/*! Index Format of Linear Index Header
|
||||
*/
|
||||
struct LinearIndexHeader {
|
||||
LinearIndexHeader()
|
||||
: header_size(0),
|
||||
total_vector_count(0),
|
||||
linear_body_size(0),
|
||||
linear_list_count(0),
|
||||
block_vector_count(0),
|
||||
block_size(0),
|
||||
block_count(0),
|
||||
index_meta_size(0) {
|
||||
memset(reserved_, 0, sizeof(reserved_));
|
||||
memset(index_meta, 0, sizeof(index_meta));
|
||||
}
|
||||
uint32_t header_size{0};
|
||||
uint32_t total_vector_count{0};
|
||||
uint64_t linear_body_size{0};
|
||||
uint32_t linear_list_count{0};
|
||||
uint32_t block_vector_count{0};
|
||||
uint32_t block_size{0};
|
||||
uint32_t block_count{0};
|
||||
uint32_t index_meta_size{0};
|
||||
char reserved_[28] = {0};
|
||||
char index_meta[0];
|
||||
};
|
||||
|
||||
/*! Index Format of Linear Index Meta for each Linear list
|
||||
*/
|
||||
struct LinearListMeta {
|
||||
LinearListMeta() : offset(0), block_count(0), vector_count(0), id_offset(0) {
|
||||
memset(reserved_, 0, sizeof(reserved_));
|
||||
}
|
||||
uint64_t offset{0};
|
||||
uint32_t block_count{0};
|
||||
uint32_t vector_count{0};
|
||||
uint32_t id_offset{0};
|
||||
char reserved_[16] = {0};
|
||||
};
|
||||
|
||||
/*! Index Format of Location in Linear Index for each vector
|
||||
*/
|
||||
struct LinearVecLocation {
|
||||
LinearVecLocation(size_t off, bool col)
|
||||
: offset(off), column_major(col), reserved(0u) {}
|
||||
|
||||
uint64_t offset : 48; // feature offset in posting block segment
|
||||
uint64_t column_major : 1; // coloum major if true
|
||||
uint64_t reserved : 15;
|
||||
};
|
||||
|
||||
/*! Index Format of Integer Quantizer params for each linear list
|
||||
*/
|
||||
struct LinearIntegerQuantizerParams {
|
||||
float scale{1.0};
|
||||
float bias{0.0};
|
||||
};
|
||||
|
||||
/*! Location of Vectors Block in Storage Segment
|
||||
*/
|
||||
struct BlockLocation {
|
||||
uint16_t segment_id{0};
|
||||
uint16_t block_index{0};
|
||||
};
|
||||
|
||||
/*! The Header of a Block in Storage Segment
|
||||
*/
|
||||
struct BlockHeader {
|
||||
BlockHeader() : vector_count(0u), column_major(0u), reserved(0u) {}
|
||||
BlockLocation next;
|
||||
uint16_t vector_count{0};
|
||||
uint16_t column_major : 1;
|
||||
uint16_t reserved : 15;
|
||||
};
|
||||
|
||||
struct DeletionMap {
|
||||
void set(uint32_t index) {
|
||||
bitset.set(index);
|
||||
}
|
||||
|
||||
void reset(uint32_t index) {
|
||||
bitset.reset(index);
|
||||
}
|
||||
|
||||
bool test(uint32_t index) const {
|
||||
return bitset.test(index);
|
||||
}
|
||||
|
||||
bool is_dirty() const {
|
||||
return bitset.test_any();
|
||||
}
|
||||
|
||||
ailego::FixedBitset<32> bitset{};
|
||||
};
|
||||
|
||||
static_assert(sizeof(DeletionMap) == 4, "DeletionMap must be 4 bytes");
|
||||
|
||||
/*! Meta Information of Streamer Entity
|
||||
*/
|
||||
struct StreamerLinearMeta {
|
||||
StreamerLinearMeta()
|
||||
: create_time(0),
|
||||
update_time(0),
|
||||
revision_id(0),
|
||||
segment_count(0),
|
||||
segment_size(0) {
|
||||
memset(reserved_, 0, sizeof(reserved_));
|
||||
}
|
||||
uint64_t create_time{0};
|
||||
uint64_t update_time{0};
|
||||
uint64_t revision_id{0};
|
||||
uint32_t segment_count{0};
|
||||
uint32_t segment_size{0};
|
||||
uint8_t reserved_[32] = {0};
|
||||
LinearIndexHeader header;
|
||||
};
|
||||
|
||||
/*! Location of Vector in Storage Segment
|
||||
*/
|
||||
struct VectorLocation {
|
||||
//! Constructor
|
||||
VectorLocation(void)
|
||||
: segment_id(0u), column_major(0u), reserved(0u), offset(0u) {}
|
||||
|
||||
//! Constructor
|
||||
VectorLocation(uint16_t id, bool col, uint32_t off)
|
||||
: segment_id(id), column_major(col), reserved(0u), offset(off) {}
|
||||
|
||||
uint16_t segment_id{0};
|
||||
uint16_t column_major : 1;
|
||||
uint16_t reserved : 15;
|
||||
uint32_t offset{0};
|
||||
|
||||
public:
|
||||
bool operator==(const VectorLocation &other) const {
|
||||
return segment_id == other.segment_id &&
|
||||
column_major == other.column_major && offset == other.offset;
|
||||
}
|
||||
};
|
||||
|
||||
static_assert(sizeof(VectorLocation) == sizeof(uint64_t),
|
||||
"VectorLocation must be size of 8 bytes");
|
||||
|
||||
struct KeyInfo {
|
||||
KeyInfo(void) : centroid_idx(0u) {}
|
||||
KeyInfo(uint32_t idx, const VectorLocation &loc)
|
||||
: centroid_idx(idx), location(loc) {}
|
||||
KeyInfo(VectorLocation loc) : location(loc) {}
|
||||
uint32_t centroid_idx{0};
|
||||
VectorLocation location;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,252 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "flat_searcher.h"
|
||||
#include <framework/index_helper.h>
|
||||
#include <framework/index_searcher.h>
|
||||
#include "flat_distance_matrix.h"
|
||||
#include "flat_searcher_context.h"
|
||||
#include "flat_searcher_provider.h"
|
||||
#include "flat_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
IndexProvider::Pointer FlatSearcher<BATCH_SIZE>::create_provider(void) const {
|
||||
std::lock_guard<std::mutex> lock(mapping_mutex_);
|
||||
|
||||
if (mapping_.empty()) {
|
||||
auto mapping_segment = container_->get(PARAM_FLAT_SEGMENT_MAPPING);
|
||||
if (!mapping_segment) {
|
||||
LOG_ERROR("Failed to fetch segment %s",
|
||||
PARAM_FLAT_SEGMENT_MAPPING.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (mapping_segment->data_size() % sizeof(uint32_t) != 0) {
|
||||
LOG_ERROR("Invalid data size %zu of mapping segment",
|
||||
mapping_segment->data_size());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
size_t mapping_count = mapping_segment->data_size() / sizeof(uint32_t);
|
||||
if (mapping_count * meta_.element_size() !=
|
||||
features_segment_->data_size()) {
|
||||
LOG_ERROR("Invalid data size %zd of mapping segment",
|
||||
features_segment_->data_size());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const uint32_t *mapping = nullptr;
|
||||
if (mapping_segment->read(0, reinterpret_cast<const void **>(&mapping),
|
||||
mapping_segment->data_size()) !=
|
||||
mapping_segment->data_size()) {
|
||||
LOG_ERROR("Failed to read data (%zu bytes) from mapping segment",
|
||||
mapping_segment->data_size());
|
||||
return nullptr;
|
||||
}
|
||||
mapping_.clear();
|
||||
mapping_.reserve(mapping_count);
|
||||
std::copy(mapping, mapping + mapping_count, std::back_inserter(mapping_));
|
||||
}
|
||||
return IndexProvider::Pointer(new (std::nothrow)
|
||||
FlatSearcherProvider<BATCH_SIZE>(this));
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatSearcher<BATCH_SIZE>::load(IndexStorage::Pointer cntr,
|
||||
IndexMetric::Pointer measure) {
|
||||
ailego::ElapsedTime stamp;
|
||||
if (!cntr) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
int error_code = IndexHelper::DeserializeFromStorage(cntr.get(), &meta_);
|
||||
if (error_code != 0) {
|
||||
LOG_ERROR(
|
||||
"Failed to deserialize index meta from container %s, error=%d, %s",
|
||||
cntr->name().c_str(), error_code, IndexError::What(error_code));
|
||||
return error_code;
|
||||
}
|
||||
|
||||
if (!measure) {
|
||||
error_code = InitializeMetric(meta_, &measure_);
|
||||
if (error_code != 0) {
|
||||
LOG_ERROR("Failed to initialize index measure %s, error=%d, %s",
|
||||
meta_.metric_name().c_str(), error_code,
|
||||
IndexError::What(error_code));
|
||||
return error_code;
|
||||
}
|
||||
if (measure_->query_metric()) {
|
||||
measure_ = measure_->query_metric();
|
||||
}
|
||||
} else {
|
||||
if (!measure->is_matched(meta_)) {
|
||||
LOG_ERROR(
|
||||
"The index measure is unmatched with index meta from container.");
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
measure_ = std::move(measure);
|
||||
}
|
||||
|
||||
column_major_order_ = (meta_.major_order() == IndexMeta::MO_COLUMN);
|
||||
distance_matrix_.initialize(*measure_);
|
||||
|
||||
if (column_major_order_) {
|
||||
if (!distance_matrix_.is_valid()) {
|
||||
LOG_ERROR("Lack of distance functions to support column index.");
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
} else {
|
||||
if (!distance_matrix_.is_valid(1, 1)) {
|
||||
LOG_ERROR("Lack of distance functions to support row index.");
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
}
|
||||
|
||||
auto keys_segment = cntr->get(PARAM_FLAT_SEGMENT_KEYS);
|
||||
if (!keys_segment) {
|
||||
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
features_segment_ = cntr->get(PARAM_FLAT_SEGMENT_FEATURES);
|
||||
if (!features_segment_) {
|
||||
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
if (keys_segment->data_size() % sizeof(uint64_t) != 0) {
|
||||
LOG_ERROR("Invalid data size %zu of keys segment",
|
||||
keys_segment->data_size());
|
||||
return IndexError_InvalidLength;
|
||||
}
|
||||
|
||||
size_t keys_count = keys_segment->data_size() / sizeof(uint64_t);
|
||||
if (keys_count * meta_.element_size() != features_segment_->data_size()) {
|
||||
LOG_ERROR("Invalid data size %zd of features segment",
|
||||
features_segment_->data_size());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
if (keys_segment->read(0, reinterpret_cast<const void **>(&keys_),
|
||||
keys_segment->data_size()) !=
|
||||
keys_segment->data_size()) {
|
||||
LOG_ERROR("Failed to read data (%zu bytes) from keys segment",
|
||||
keys_segment->data_size());
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < keys_count; i++) {
|
||||
key_id_mapping_[keys_[i]] = i;
|
||||
}
|
||||
|
||||
container_ = cntr;
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
stats_.set_loaded_count(keys_count);
|
||||
stats_.set_loaded_costtime(stamp.milli_seconds());
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatSearcher<BATCH_SIZE>::search_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const {
|
||||
ailego_assert(query && !!context);
|
||||
ailego_assert(measure_->is_matched(meta_, qmeta));
|
||||
|
||||
FlatSearcherContext<BATCH_SIZE> *bf_context =
|
||||
dynamic_cast<FlatSearcherContext<BATCH_SIZE> *>(context.get());
|
||||
if (!bf_context) {
|
||||
LOG_ERROR("Invalid brute-force searcher context");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (bf_context->magic() != magic_) {
|
||||
bf_context->reset(this);
|
||||
}
|
||||
if (bf_context->group_by_search()) {
|
||||
return bf_context->group_by_search_impl(query, qmeta, 1);
|
||||
} else {
|
||||
return (column_major_order_ ? bf_context->search_column(query, qmeta)
|
||||
: bf_context->search_row(query, qmeta));
|
||||
}
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatSearcher<BATCH_SIZE>::search_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
ailego_assert(query && count && !!context);
|
||||
ailego_assert(measure_->is_matched(meta_, qmeta));
|
||||
|
||||
FlatSearcherContext<BATCH_SIZE> *bf_context =
|
||||
dynamic_cast<FlatSearcherContext<BATCH_SIZE> *>(context.get());
|
||||
if (!bf_context) {
|
||||
LOG_ERROR("Invalid brute-force searcher context");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (bf_context->magic() != magic_) {
|
||||
bf_context->reset(this);
|
||||
}
|
||||
|
||||
if (bf_context->group_by_search()) {
|
||||
return bf_context->group_by_search_impl(query, qmeta, count);
|
||||
} else {
|
||||
return (column_major_order_ ? bf_context->search_column(query, qmeta, count)
|
||||
: bf_context->search_row(query, qmeta, count));
|
||||
}
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatSearcher<BATCH_SIZE>::search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
ailego_assert(query && count && !!context);
|
||||
ailego_assert(measure_->is_matched(meta_, qmeta));
|
||||
|
||||
if (ailego_unlikely(p_keys.size() != count)) {
|
||||
LOG_ERROR("The size of p_keys is not equal to count");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
FlatSearcherContext<BATCH_SIZE> *bf_context =
|
||||
dynamic_cast<FlatSearcherContext<BATCH_SIZE> *>(context.get());
|
||||
if (!bf_context) {
|
||||
LOG_ERROR("Invalid brute-force searcher context");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (bf_context->magic() != magic_) {
|
||||
bf_context->reset(this);
|
||||
}
|
||||
|
||||
return bf_context->search_bf_by_p_keys_impl(query, p_keys, qmeta, count);
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
IndexSearcher::Context::Pointer FlatSearcher<BATCH_SIZE>::create_context(
|
||||
void) const {
|
||||
return IndexSearcher::Context::Pointer(
|
||||
new FlatSearcherContext<BATCH_SIZE>(this));
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_SEARCHER_ALIAS(LinearSearcher, FlatSearcher<32>);
|
||||
INDEX_FACTORY_REGISTER_SEARCHER_ALIAS(FlatSearcher, FlatSearcher<32>);
|
||||
INDEX_FACTORY_REGISTER_SEARCHER_ALIAS(FlatSearcher16, FlatSearcher<16>);
|
||||
INDEX_FACTORY_REGISTER_SEARCHER_ALIAS(FlatSearcher32, FlatSearcher<32>);
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,182 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/container/params.h>
|
||||
#include "framework/index_searcher.h"
|
||||
#include "flat_distance_matrix.h"
|
||||
#include "flat_index_format.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Flat Searcher
|
||||
*/
|
||||
template <size_t BATCH_SIZE>
|
||||
class FlatSearcher : public IndexSearcher {
|
||||
public:
|
||||
//! Destructor
|
||||
virtual ~FlatSearcher(void) = default;
|
||||
|
||||
//! Initialize Searcher
|
||||
int init(const ailego::Params &index_params) override {
|
||||
params_ = index_params;
|
||||
read_block_size_ = FLAT_READ_BLOCK_SIZE;
|
||||
index_params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cleanup Searcher
|
||||
int cleanup(void) override {
|
||||
return this->unload();
|
||||
}
|
||||
|
||||
//! Load index from container
|
||||
int load(IndexStorage::Pointer cntr, IndexMetric::Pointer measure) override;
|
||||
|
||||
//! Unload index
|
||||
int unload(void) override {
|
||||
container_ = nullptr;
|
||||
measure_ = nullptr;
|
||||
features_segment_ = nullptr;
|
||||
keys_ = nullptr;
|
||||
key_id_mapping_.clear();
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Similarity brute force search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override {
|
||||
return this->search_impl(query, qmeta, context);
|
||||
}
|
||||
|
||||
//! Similarity brute force search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override {
|
||||
return this->search_impl(query, qmeta, count, context);
|
||||
}
|
||||
|
||||
//! Similarity search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Retrieve statistics
|
||||
const IndexSearcher::Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
//! Retrieve params of index
|
||||
const ailego::Params ¶ms(void) const override {
|
||||
return params_;
|
||||
}
|
||||
|
||||
//! Create a searcher context
|
||||
IndexSearcher::Context::Pointer create_context(void) const override;
|
||||
|
||||
//! Create a searcher provider
|
||||
IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Retrieve magic number
|
||||
uint32_t magic(void) const {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
//! Retrieve block size of data read
|
||||
uint32_t read_block_size(void) const {
|
||||
return read_block_size_;
|
||||
}
|
||||
|
||||
//! Retrieve primary key via index id
|
||||
uint64_t key(size_t i) const {
|
||||
return keys_[i];
|
||||
}
|
||||
|
||||
// Retrieve index id via primary key
|
||||
node_id_t get_id(key_t key) const {
|
||||
auto it = key_id_mapping_.find(key);
|
||||
if (it != key_id_mapping_.end()) {
|
||||
return it->second;
|
||||
} else {
|
||||
return kInvalidNodeId;
|
||||
}
|
||||
}
|
||||
|
||||
//! Retrieve primary key via index id
|
||||
uint32_t local_index(size_t i) const {
|
||||
return mapping_[i];
|
||||
}
|
||||
|
||||
//! Retrieve primary key via index id
|
||||
inline bool column_major_order(void) const {
|
||||
return column_major_order_;
|
||||
}
|
||||
|
||||
//! Retrieve the distance matrix
|
||||
const FlatDistanceMatrix<BATCH_SIZE> &distance_matrix(void) const {
|
||||
return distance_matrix_;
|
||||
}
|
||||
|
||||
//! Clone a features segment
|
||||
IndexStorage::Segment::Pointer clone_features_segment(void) const {
|
||||
return features_segment_->clone();
|
||||
}
|
||||
|
||||
const void *get_vector(key_t key) const override {
|
||||
auto provider = this->create_provider();
|
||||
return provider->get_vector(key);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
const uint64_t *keys_{nullptr};
|
||||
std::map<key_t, node_id_t> key_id_mapping_;
|
||||
uint32_t magic_{IndexContext::GenerateMagic()};
|
||||
uint32_t read_block_size_{FLAT_READ_BLOCK_SIZE};
|
||||
bool column_major_order_{false};
|
||||
IndexMeta meta_{};
|
||||
IndexStorage::Pointer container_{};
|
||||
IndexMetric::Pointer measure_{};
|
||||
ailego::Params params_{};
|
||||
IndexStorage::Segment::Pointer features_segment_{};
|
||||
mutable std::vector<uint32_t> mapping_{};
|
||||
mutable std::mutex mapping_mutex_{};
|
||||
FlatDistanceMatrix<BATCH_SIZE> distance_matrix_{};
|
||||
IndexSearcher::Stats stats_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,220 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "flat_distance_matrix.h"
|
||||
#include "flat_searcher.h"
|
||||
// #include "flat_streamer.h"
|
||||
#include "flat_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Brute Force Searcher Provider
|
||||
*/
|
||||
template <size_t BATCH_SIZE>
|
||||
class FlatSearcherProvider : public IndexProvider {
|
||||
public:
|
||||
//! Constructor
|
||||
FlatSearcherProvider(const FlatSearcher<BATCH_SIZE> *owner) {
|
||||
feature_size_ = owner->meta().element_size();
|
||||
features_segment_ = owner->clone_features_segment();
|
||||
total_vector_count_ =
|
||||
features_segment_->data_size() / owner->meta().element_size();
|
||||
owner_ = owner;
|
||||
block_buffer_.resize(BATCH_SIZE * feature_size_);
|
||||
}
|
||||
|
||||
//! Create a new iterator
|
||||
IndexProvider::Iterator::Pointer create_iterator(void) override {
|
||||
return IndexProvider::Iterator::Pointer(
|
||||
new (std::nothrow) FlatSearcherProvider::Iterator(owner_));
|
||||
}
|
||||
|
||||
//! Retrieve count of vectors
|
||||
size_t count(void) const override {
|
||||
return total_vector_count_;
|
||||
}
|
||||
|
||||
//! Retrieve dimension of vector
|
||||
size_t dimension(void) const override {
|
||||
return owner_->meta().dimension();
|
||||
}
|
||||
|
||||
//! Retrieve type of vector
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return owner_->meta().data_type();
|
||||
}
|
||||
|
||||
//! Retrieve vector size in bytes
|
||||
size_t element_size(void) const override {
|
||||
return owner_->meta().element_size();
|
||||
}
|
||||
|
||||
//! Retrieve a vector using a primary key
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
return this->get_vector_by_index(owner_->get_id(key));
|
||||
}
|
||||
|
||||
//! Retrieve the owner class
|
||||
const std::string &owner_class(void) const override {
|
||||
return owner_->name();
|
||||
}
|
||||
|
||||
protected:
|
||||
/*! Brute Force Provider Iterator
|
||||
*/
|
||||
class Iterator : public IndexProvider::Iterator {
|
||||
public:
|
||||
//! Constructor
|
||||
Iterator(const FlatSearcher<BATCH_SIZE> *owner) {
|
||||
block_buffer_.resize(BATCH_SIZE * owner->meta().element_size());
|
||||
feature_size_ = owner->meta().element_size();
|
||||
features_segment_ = owner->clone_features_segment();
|
||||
total_vector_count_ =
|
||||
features_segment_->data_size() / owner->meta().element_size();
|
||||
owner_ = owner;
|
||||
cursor_index_ = 0;
|
||||
offset_ = 0;
|
||||
this->next_block();
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
//! NOTICE: the vec feature will be changed after iterating to next, so
|
||||
//! the caller need to keep a copy of it before iterator to next vector
|
||||
const void *data(void) const override {
|
||||
return data_;
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
bool is_valid(void) const override {
|
||||
return (!invalid_ && cursor_index_ < total_vector_count_);
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
uint64_t key(void) const override {
|
||||
return owner_->key(cursor_index_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
void next(void) override {
|
||||
++cursor_index_;
|
||||
|
||||
if ((cursor_index_ % BATCH_SIZE) != 0) {
|
||||
data_ += feature_size_;
|
||||
} else {
|
||||
this->next_block();
|
||||
}
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Read a block of data
|
||||
void next_block(void) {
|
||||
const void *read_data = nullptr;
|
||||
size_t read_size = 0;
|
||||
|
||||
if (cursor_index_ >= total_vector_count_) {
|
||||
invalid_ = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (cursor_index_ + BATCH_SIZE < total_vector_count_) {
|
||||
read_size = BATCH_SIZE * feature_size_;
|
||||
} else {
|
||||
read_size = (total_vector_count_ - cursor_index_) * feature_size_;
|
||||
}
|
||||
if (features_segment_->read(offset_, &read_data, read_size) !=
|
||||
read_size) {
|
||||
LOG_ERROR("Failed to read data (%zu bytes) from features segment",
|
||||
read_size);
|
||||
invalid_ = true;
|
||||
return;
|
||||
}
|
||||
offset_ += read_size;
|
||||
|
||||
// The order of data may be a column format, convert it to the row format.
|
||||
if (owner_->column_major_order() &&
|
||||
read_size == BATCH_SIZE * feature_size_) {
|
||||
uint32_t align_size =
|
||||
IndexMeta::AlignSizeof(owner_->meta().data_type());
|
||||
ReverseTranspose<BATCH_SIZE>(align_size, read_data,
|
||||
feature_size_ / align_size,
|
||||
&block_buffer_[0]);
|
||||
data_ = block_buffer_.data();
|
||||
} else {
|
||||
data_ = reinterpret_cast<const uint8_t *>(read_data);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const FlatSearcher<BATCH_SIZE> *owner_{nullptr};
|
||||
IndexStorage::Segment::Pointer features_segment_{};
|
||||
uint32_t total_vector_count_{0};
|
||||
uint32_t feature_size_{0};
|
||||
std::vector<uint8_t> block_buffer_{};
|
||||
const uint8_t *data_{nullptr};
|
||||
uint64_t offset_{0};
|
||||
uint32_t cursor_index_{0};
|
||||
bool invalid_{false};
|
||||
};
|
||||
|
||||
//! Retrieve a vector via local index
|
||||
const void *get_vector_by_index(uint32_t index) const {
|
||||
const void *read_data = nullptr;
|
||||
if (index == kInvalidNodeId) {
|
||||
LOG_ERROR("Failed to get vector by Invalid Id.");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (owner_->column_major_order() &&
|
||||
index < (total_vector_count_ - (total_vector_count_ % BATCH_SIZE))) {
|
||||
uint32_t block_size = feature_size_ * BATCH_SIZE;
|
||||
uint64_t offset = (index - (index % BATCH_SIZE)) * feature_size_;
|
||||
|
||||
if (features_segment_->read(offset, &read_data, block_size) !=
|
||||
block_size) {
|
||||
LOG_ERROR("Failed to read data (%u bytes) from features segment",
|
||||
block_size);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t align_size = IndexMeta::AlignSizeof(owner_->meta().data_type());
|
||||
ReverseTranspose<BATCH_SIZE>(
|
||||
align_size, read_data, feature_size_ / align_size, &block_buffer_[0]);
|
||||
read_data = block_buffer_.data() + ((index % BATCH_SIZE) * feature_size_);
|
||||
|
||||
} else {
|
||||
if (features_segment_->read(index * feature_size_, &read_data,
|
||||
feature_size_) != feature_size_) {
|
||||
LOG_ERROR("Failed to read data (%u bytes) from features segment",
|
||||
feature_size_);
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
return read_data;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
const FlatSearcher<BATCH_SIZE> *owner_{nullptr};
|
||||
IndexStorage::Segment::Pointer features_segment_{};
|
||||
uint32_t feature_size_{0};
|
||||
uint32_t total_vector_count_{0};
|
||||
mutable std::vector<uint8_t> block_buffer_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,491 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "flat_streamer.h"
|
||||
#include <framework/index_factory.h>
|
||||
#include "flat_streamer_context.h"
|
||||
#include "flat_streamer_dumper.h"
|
||||
#include "flat_streamer_provider.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
#define WRITE_LOCK_GUARD(MUTEX, LOCK_NAME) \
|
||||
ailego::WriteLock write_lock(MUTEX); \
|
||||
std::unique_lock<ailego::WriteLock> LOCK_NAME(write_lock);
|
||||
|
||||
#define READ_LOCK_GUARD_DEFER(MUTEX, LOCK_NAME) \
|
||||
ailego::ReadLock read_lock(MUTEX); \
|
||||
std::unique_lock<ailego::ReadLock> LOCK_NAME(read_lock, std::defer_lock);
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
FlatStreamer<BATCH_SIZE>::FlatStreamer() : entity_(stats_) {}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
FlatStreamer<BATCH_SIZE>::~FlatStreamer() {
|
||||
if (state_ == STATE_INITED) {
|
||||
this->cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::init(const IndexMeta &imeta,
|
||||
const ailego::Params ¶ms) {
|
||||
meta_ = imeta;
|
||||
meta_.set_streamer("FlatStreamer", 0U, params);
|
||||
|
||||
int error_code = InitializeMetric(meta_, &metric_);
|
||||
if (error_code != 0) {
|
||||
LOG_ERROR("Failed to initialize index metric %s, error=%d, %s",
|
||||
meta_.metric_name().c_str(), error_code,
|
||||
IndexError::What(error_code));
|
||||
return error_code;
|
||||
}
|
||||
if (metric_->query_metric()) {
|
||||
metric_ = metric_->query_metric();
|
||||
}
|
||||
|
||||
// 参数设置
|
||||
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order_)) {
|
||||
meta_.set_major_order(column_major_order_ ? IndexMeta::MO_COLUMN
|
||||
: IndexMeta::MO_ROW);
|
||||
}
|
||||
// Verify column major order
|
||||
if (meta_.major_order() != IndexMeta::MO_ROW) {
|
||||
IndexMeta::DataType ft = meta_.data_type();
|
||||
|
||||
bool support_column_major = true;
|
||||
if ((ft != IndexMeta::DT_FP32 && ft != IndexMeta::DT_FP16 &&
|
||||
ft != IndexMeta::DT_INT8 && ft != IndexMeta::DT_INT4 &&
|
||||
ft != IndexMeta::DT_BINARY32 && ft != IndexMeta::DT_BINARY64) ||
|
||||
(meta_.unit_size() != IndexMeta::UnitSizeof(ft))) {
|
||||
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
|
||||
LOG_ERROR("Unsupported type %d with unit size %u.", ft,
|
||||
meta_.unit_size());
|
||||
return IndexError_Unsupported;
|
||||
} else {
|
||||
support_column_major = false;
|
||||
}
|
||||
}
|
||||
if (meta_.element_size() % IndexMeta::AlignSizeof(ft) != 0) {
|
||||
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
|
||||
LOG_ERROR("Unsupported type %d with dimension %u.", ft,
|
||||
meta_.dimension());
|
||||
return IndexError_Unsupported;
|
||||
} else {
|
||||
support_column_major = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (meta_.major_order() == IndexMeta::MO_UNDEFINED &&
|
||||
support_column_major) {
|
||||
meta_.set_major_order(IndexMeta::MO_ROW);
|
||||
}
|
||||
}
|
||||
|
||||
if (!VerifyMetric(meta_)) {
|
||||
LOG_ERROR("Invalid index metric %s.", meta_.metric_name().c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
read_block_size_ = FLAT_READ_BLOCK_SIZE;
|
||||
params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
|
||||
params.get(FLAT_SEARCHER_USE_ID_MAP, &use_key_info_map_);
|
||||
|
||||
// entity init
|
||||
uint32_t block_vector_count = kDefaultBlockVecCount;
|
||||
uint32_t segment_size = kDefaultSegmentSize;
|
||||
bool filter_same_key = true;
|
||||
entity_.set_block_vector_count(block_vector_count);
|
||||
entity_.set_segment_size(segment_size);
|
||||
entity_.enable_filter_same_key(filter_same_key);
|
||||
entity_.set_linear_list_count(1);
|
||||
entity_.set_use_key_info_map(use_key_info_map_);
|
||||
*entity_.mutable_meta() = meta_;
|
||||
|
||||
state_ = STATE_INITED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::cleanup() {
|
||||
if (state_ == STATE_OPENED) {
|
||||
this->close();
|
||||
}
|
||||
|
||||
LOG_DEBUG("FlatStreamer cleanup");
|
||||
state_ = STATE_INIT;
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::open(IndexStorage::Pointer stg) {
|
||||
if (!stg) {
|
||||
LOG_ERROR("Failed to open for invalid storage");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (ailego_unlikely(state_ != STATE_INITED)) {
|
||||
LOG_ERROR("Open storage failed, init streamer first!");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
LOG_DEBUG("FlatStreamer open with %s", stg->name().c_str());
|
||||
|
||||
int ret = entity_.open(std::move(stg), meta_);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Failed to open storage");
|
||||
return ret;
|
||||
}
|
||||
magic_ = IndexContext::GenerateMagic();
|
||||
|
||||
state_ = STATE_OPENED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::close(void) {
|
||||
LOG_DEBUG("FlatStreamer close");
|
||||
|
||||
entity_.flush_linear_meta();
|
||||
|
||||
stats_.clear();
|
||||
|
||||
int ret = entity_.close();
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
state_ = STATE_INITED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::flush(uint64_t checkpoint) {
|
||||
LOG_INFO("FlatStreamer flush with checkpoint %zu", (size_t)checkpoint);
|
||||
return entity_.flush(checkpoint);
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::dump(const IndexDumper::Pointer &dumper) {
|
||||
std::string searcher_name = "FlatSearcher";
|
||||
if (BATCH_SIZE == 16) {
|
||||
searcher_name = "FlatSearcher16";
|
||||
}
|
||||
meta_.set_searcher(searcher_name, 0U, ailego::Params());
|
||||
WRITE_LOCK_GUARD(dump_mutex_, dump_lock);
|
||||
std::shared_ptr<FlatStreamerDumper<BATCH_SIZE>> bf_dumper =
|
||||
std::make_shared<FlatStreamerDumper<BATCH_SIZE>>(this);
|
||||
int ret = bf_dumper->dump(dumper);
|
||||
*(stats_.mutable_dumped_size()) += bf_dumper->dump_size();
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
IndexStreamer::Context::UPointer FlatStreamer<BATCH_SIZE>::create_context(
|
||||
void) const {
|
||||
if (state_ != STATE_OPENED) {
|
||||
LOG_ERROR("Failed to create Context, open storage first!");
|
||||
return Context::UPointer();
|
||||
}
|
||||
return IndexStreamer::Context::Pointer(
|
||||
new FlatStreamerContext<BATCH_SIZE>(this));
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
IndexProvider::Pointer FlatStreamer<BATCH_SIZE>::create_provider(void) const {
|
||||
return IndexProvider::Pointer(new (std::nothrow)
|
||||
FlatStreamerProvider<BATCH_SIZE>(this));
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::add_impl(uint64_t pkey, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::UPointer &context) {
|
||||
if (!query || qmeta.dimension() != meta_.dimension() ||
|
||||
qmeta.data_type() != meta_.data_type() ||
|
||||
qmeta.element_size() != meta_.element_size()) {
|
||||
LOG_ERROR(
|
||||
"Failed to add for invalid arguments, query=%p, qmeta(type=%u "
|
||||
"dim=%u size=%u) vs meta(type=%u dim=%u size=%u)",
|
||||
query, qmeta.data_type(), qmeta.dimension(), qmeta.element_size(),
|
||||
meta_.data_type(), meta_.dimension(), meta_.element_size());
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
auto *ctx = dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
|
||||
if (!ctx) {
|
||||
LOG_ERROR("Failed to cast FlatStreamerContext");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
READ_LOCK_GUARD_DEFER(dump_mutex_, dump_lock);
|
||||
|
||||
if (!dump_lock.try_lock()) {
|
||||
LOG_ERROR("Cannot add vector while dumping index");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
// IndexQueryMeta iv_qmeta;
|
||||
// int ret = entity_.convert(query, qmeta, &query, &iv_qmeta);
|
||||
// if (ret != 0) {
|
||||
// LOG_ERROR("Failed to convert record for %s",
|
||||
// IndexError::What(ret));
|
||||
// (*stats_.mutable_discarded_count())++;
|
||||
// return ret;
|
||||
// }
|
||||
|
||||
int ret = entity_.add(pkey, query, qmeta.element_size());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to add record for %s", IndexError::What(ret));
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) {
|
||||
if (!query || qmeta.dimension() != meta_.dimension() ||
|
||||
qmeta.data_type() != meta_.data_type() ||
|
||||
qmeta.element_size() != meta_.element_size()) {
|
||||
LOG_ERROR(
|
||||
"Failed to add for invalid arguments, query=%p, qmeta(type=%u "
|
||||
"dim=%u size=%u) vs meta(type=%u dim=%u size=%u)",
|
||||
query, qmeta.data_type(), qmeta.dimension(), qmeta.element_size(),
|
||||
meta_.data_type(), meta_.dimension(), meta_.element_size());
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
auto *ctx = dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
|
||||
if (!ctx) {
|
||||
LOG_ERROR("Failed to cast FlatStreamerContext");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Cast;
|
||||
}
|
||||
|
||||
READ_LOCK_GUARD_DEFER(dump_mutex_, dump_lock);
|
||||
|
||||
if (!dump_lock.try_lock()) {
|
||||
LOG_ERROR("Cannot add vector while dumping index");
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
int ret = entity_.add_vector_with_id(id, query, qmeta.element_size());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to add record for %s", IndexError::What(ret));
|
||||
(*stats_.mutable_discarded_count())++;
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::search_bf_impl(const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
ailego_assert(query && count && !!context);
|
||||
ailego_assert(metric_->is_matched(meta_, qmeta));
|
||||
|
||||
FlatStreamerContext<BATCH_SIZE> *bf_context =
|
||||
dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
|
||||
if (!bf_context) {
|
||||
LOG_ERROR("Invalid brute-force streamer context");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (bf_context->magic() != magic_) {
|
||||
bf_context->reset(this);
|
||||
}
|
||||
|
||||
if (bf_context->group_by_search()) {
|
||||
return group_by_search_impl(query, qmeta, count, context);
|
||||
}
|
||||
|
||||
bf_context->reset_results(count);
|
||||
auto &filter = bf_context->filter();
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
auto *heap = bf_context->result_heap();
|
||||
auto *context_stats = bf_context->mutable_stats(q);
|
||||
uint32_t scan_count = 0;
|
||||
int ret = entity_.search(query, filter, &scan_count, heap, context_stats);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Failed to search for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
heap->sort();
|
||||
bf_context->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
ailego_assert(query && count && !!context);
|
||||
ailego_assert(metric_->is_matched(meta_, qmeta));
|
||||
|
||||
FlatStreamerContext<BATCH_SIZE> *bf_context =
|
||||
dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
|
||||
if (!bf_context) {
|
||||
LOG_ERROR("Invalid brute-force streamer context");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (bf_context->magic() != magic_) {
|
||||
bf_context->reset(this);
|
||||
}
|
||||
|
||||
if (bf_context->group_by_search()) {
|
||||
return group_by_search_p_keys_impl(query, p_keys, qmeta, count, context);
|
||||
}
|
||||
|
||||
bf_context->reset_results(count);
|
||||
auto &filter = bf_context->filter();
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
auto *heap = bf_context->result_heap();
|
||||
for (node_id_t idx = 0; idx < p_keys[q].size(); ++idx) {
|
||||
uint64_t key = p_keys[q][idx];
|
||||
if (!filter.is_valid() || !filter(key)) {
|
||||
dist_t dist = 0;
|
||||
IndexStorage::MemoryBlock block;
|
||||
entity_.get_vector_by_key(key, block);
|
||||
entity_.row_major_distance(query, block.data(), 1, &dist);
|
||||
heap->emplace(key, dist);
|
||||
}
|
||||
}
|
||||
heap->sort();
|
||||
bf_context->topk_to_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::group_by_search_impl(
|
||||
const void *query, const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
FlatStreamerContext<BATCH_SIZE> *bf_context =
|
||||
dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
|
||||
if (!bf_context) {
|
||||
LOG_ERROR("Invalid brute-force streamer context");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
bf_context->resize_group_results(count);
|
||||
if (!bf_context->group_by().is_valid()) {
|
||||
LOG_ERROR("Invalid group-by function");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::function<std::string(uint64_t)> group_by = [&](uint64_t key) {
|
||||
return bf_context->group_by()(key);
|
||||
};
|
||||
|
||||
auto iterator = entity_.creater_iterator();
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
bf_context->group_topk_heaps().clear();
|
||||
for (node_id_t id = 0; id < entity_.vector_count(); ++id) {
|
||||
uint64_t key = entity_.key(id);
|
||||
if (!bf_context->filter().is_valid() || !bf_context->filter()(key)) {
|
||||
dist_t dist = 0;
|
||||
IndexStorage::MemoryBlock block;
|
||||
entity_.get_vector_by_key(key, block);
|
||||
entity_.row_major_distance(query, block.data(), 1, &dist);
|
||||
|
||||
std::string group_id = group_by(key);
|
||||
auto &topk_heap = bf_context->group_topk_heaps()[group_id];
|
||||
if (topk_heap.empty()) {
|
||||
topk_heap.limit(bf_context->group_topk());
|
||||
}
|
||||
topk_heap.emplace(key, dist);
|
||||
}
|
||||
}
|
||||
bf_context->topk_to_group_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
int FlatStreamer<BATCH_SIZE>::group_by_search_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const {
|
||||
FlatStreamerContext<BATCH_SIZE> *bf_context =
|
||||
dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
|
||||
if (!bf_context) {
|
||||
LOG_ERROR("Invalid brute-force streamer context");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
bf_context->resize_group_results(count);
|
||||
if (!bf_context->group_by().is_valid()) {
|
||||
LOG_ERROR("Invalid group-by function");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
std::function<std::string(uint64_t)> group_by = [&](uint64_t key) {
|
||||
return bf_context->group_by()(key);
|
||||
};
|
||||
|
||||
auto iterator = entity_.creater_iterator();
|
||||
|
||||
for (size_t q = 0; q < count; ++q) {
|
||||
bf_context->group_topk_heaps().clear();
|
||||
for (node_id_t idx = 0; idx < p_keys[q].size(); ++idx) {
|
||||
uint64_t key = p_keys[q][idx];
|
||||
if (!bf_context->filter().is_valid() || !bf_context->filter()(key)) {
|
||||
dist_t dist = 0;
|
||||
IndexStorage::MemoryBlock block;
|
||||
entity_.get_vector_by_key(key, block);
|
||||
entity_.row_major_distance(query, block.data(), 1, &dist);
|
||||
|
||||
std::string group_id = group_by(key);
|
||||
auto &topk_heap = bf_context->group_topk_heaps()[group_id];
|
||||
if (topk_heap.empty()) {
|
||||
topk_heap.limit(bf_context->group_topk());
|
||||
}
|
||||
topk_heap.emplace(key, dist);
|
||||
}
|
||||
}
|
||||
bf_context->topk_to_group_result(q);
|
||||
query = static_cast<const char *>(query) + qmeta.element_size();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_STREAMER_ALIAS(LinearStreamer, FlatStreamer<32>);
|
||||
INDEX_FACTORY_REGISTER_STREAMER_ALIAS(FlatStreamer, FlatStreamer<32>);
|
||||
INDEX_FACTORY_REGISTER_STREAMER_ALIAS(FlatStreamer16, FlatStreamer<16>);
|
||||
INDEX_FACTORY_REGISTER_STREAMER_ALIAS(FlatStreamer32, FlatStreamer<32>);
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,190 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <framework/index_streamer.h>
|
||||
#include "flat_streamer_entity.h"
|
||||
#include "flat_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Flat Streamer
|
||||
*/
|
||||
template <size_t BATCH_SIZE>
|
||||
class FlatStreamer : public IndexStreamer {
|
||||
public:
|
||||
using ContextPointer = IndexStreamer::Context::UPointer;
|
||||
|
||||
FlatStreamer(void);
|
||||
virtual ~FlatStreamer(void);
|
||||
|
||||
FlatStreamer(const FlatStreamer &streamer) = delete;
|
||||
FlatStreamer &operator=(const FlatStreamer &streamer) = delete;
|
||||
|
||||
public:
|
||||
//! Initialize Streamer
|
||||
int init(const IndexMeta &, const ailego::Params &) override;
|
||||
|
||||
//! Cleanup Streamer
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Create a context
|
||||
IndexStreamer::Context::UPointer create_context(void) const override;
|
||||
|
||||
//! Create a new iterator
|
||||
IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Add a vector into index
|
||||
int add_impl(uint64_t pkey, const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::UPointer &context) override;
|
||||
|
||||
int add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
//! Similarity search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::UPointer &context) const override {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Similarity search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::UPointer &context) const override {
|
||||
return search_bf_impl(query, qmeta, count, context);
|
||||
}
|
||||
|
||||
//! Similarity brute force search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::UPointer &context) const override {
|
||||
return search_bf_impl(query, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Similarity brute force search
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::UPointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::UPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::UPointer &context) const override;
|
||||
|
||||
int group_by_search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::UPointer &context) const;
|
||||
|
||||
int group_by_search_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
Context::Pointer &context) const;
|
||||
|
||||
//! Open index from file path
|
||||
int open(IndexStorage::Pointer stg) override;
|
||||
|
||||
//! Close file
|
||||
int close(void) override;
|
||||
|
||||
//! flush file
|
||||
int flush(uint64_t checkpoint) override;
|
||||
|
||||
//! Dump index into storage
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
const FlatStreamerEntity &entity(void) const {
|
||||
return entity_;
|
||||
}
|
||||
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
return this->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
virtual int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return this->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
const void *get_vector_by_key(uint64_t key) const {
|
||||
return entity_.get_vector_by_key(key);
|
||||
}
|
||||
|
||||
int get_vector_by_key(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_.get_vector_by_key(key, block);
|
||||
}
|
||||
const void *get_vector_by_id(uint32_t id) const override {
|
||||
return get_vector_by_key(id);
|
||||
}
|
||||
|
||||
int get_vector_by_id(const uint32_t id,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return get_vector_by_key(id, block);
|
||||
}
|
||||
|
||||
uint32_t magic(void) const {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
//! Retrieve block size of data read
|
||||
uint32_t read_block_size(void) const {
|
||||
return read_block_size_;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Constants
|
||||
static constexpr uint32_t kDefaultBlockVecCount = 32u;
|
||||
static constexpr uint32_t kDefaultSegmentSize = 4 * 1024 * 1024u;
|
||||
static constexpr float kDefaultDocsSoftLimitRatio = 0.9f;
|
||||
|
||||
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_OPENED = 2 };
|
||||
|
||||
//! Members
|
||||
uint32_t magic_{0};
|
||||
uint32_t docs_hard_limit_{std::numeric_limits<uint32_t>::max()};
|
||||
uint32_t docs_soft_limit_{0};
|
||||
IndexMeta meta_{};
|
||||
std::vector<std::vector<std::string>> data_;
|
||||
IndexStreamer::Stats stats_{};
|
||||
IndexMetric::Pointer metric_{};
|
||||
State state_{STATE_INIT};
|
||||
mutable std::mutex mapping_mutex_{};
|
||||
ailego::SharedMutex dump_mutex_{};
|
||||
FlatStreamerEntity entity_;
|
||||
bool column_major_order_{false};
|
||||
bool use_key_info_map_{true};
|
||||
uint32_t read_block_size_{0};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,244 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "flat_streamer.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Brute Force Streamer Context
|
||||
*/
|
||||
template <size_t BATCH_SIZE>
|
||||
class FlatStreamerContext : public IndexStreamer::Context {
|
||||
public:
|
||||
//! Constructor
|
||||
FlatStreamerContext(const FlatStreamer<BATCH_SIZE> *owner) {
|
||||
this->reset(owner);
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
virtual ~FlatStreamerContext(void) = default;
|
||||
|
||||
//! Set topk of search result
|
||||
void set_topk(uint32_t topk) override {
|
||||
topk_ = topk;
|
||||
result_heap_.limit(topk);
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search result with index
|
||||
const IndexDocumentList &result(size_t idx) const override {
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve result object for output
|
||||
IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
inline IndexDocumentHeap *result_heap() {
|
||||
return &result_heap_;
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(void) const override {
|
||||
return group_results_[0];
|
||||
}
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(
|
||||
size_t idx) const override {
|
||||
return group_results_[idx];
|
||||
}
|
||||
|
||||
//! Update the parameters of context
|
||||
int update(const ailego::Params & /*params*/) override {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Retrieve magic number
|
||||
uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
//! Get group topk
|
||||
inline uint32_t group_topk() const {
|
||||
return group_topk_;
|
||||
}
|
||||
//! Get group num
|
||||
inline uint32_t group_num() const {
|
||||
return group_num_;
|
||||
}
|
||||
inline std::map<std::string, TopkHeap> &group_topk_heaps() {
|
||||
return group_topk_heaps_;
|
||||
}
|
||||
void set_fetch_vector(bool v) override {
|
||||
fetch_vector_ = v;
|
||||
}
|
||||
bool fetch_vector() const override {
|
||||
return fetch_vector_;
|
||||
}
|
||||
inline void resize_group_results(size_t size) {
|
||||
if (group_by_search()) {
|
||||
group_results_.resize(size);
|
||||
}
|
||||
}
|
||||
|
||||
void topk_to_result(uint32_t idx) {
|
||||
if (ailego_unlikely(result_heap_.size() == 0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
int size = std::min(topk_, static_cast<uint32_t>(result_heap_.size()));
|
||||
result_heap_.sort();
|
||||
results_[idx].clear();
|
||||
for (int i = 0; i < size; ++i) {
|
||||
auto score = result_heap_[i].score();
|
||||
if (score > this->threshold()) {
|
||||
break;
|
||||
}
|
||||
|
||||
key_t key = result_heap_[i].key();
|
||||
if (fetch_vector_) {
|
||||
IndexStorage::MemoryBlock block;
|
||||
owner_->entity().get_vector_by_key(key, block);
|
||||
results_[idx].emplace_back(key, score, key, block);
|
||||
} else {
|
||||
results_[idx].emplace_back(key, score);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void topk_to_group_result(uint32_t idx) {
|
||||
ailego_assert_with(idx < group_results_.size(), "invalid idx");
|
||||
group_results_[idx].clear();
|
||||
std::vector<std::pair<std::string, TopkHeap>> group_topk_list;
|
||||
std::vector<std::pair<std::string, float>> best_score_in_groups;
|
||||
for (auto itr = group_topk_heaps_.begin(); itr != group_topk_heaps_.end();
|
||||
itr++) {
|
||||
const std::string &group_id = (*itr).first;
|
||||
auto &heap = (*itr).second;
|
||||
heap.sort();
|
||||
if (heap.size() > 0) {
|
||||
float best_score = heap[0].second;
|
||||
best_score_in_groups.push_back(std::make_pair(group_id, best_score));
|
||||
}
|
||||
}
|
||||
std::sort(best_score_in_groups.begin(), best_score_in_groups.end(),
|
||||
[](const std::pair<std::string, float> &a,
|
||||
const std::pair<std::string, float> &b) -> int {
|
||||
return a.second < b.second;
|
||||
});
|
||||
// truncate to group num
|
||||
for (uint32_t i = 0; i < group_num() && i < best_score_in_groups.size();
|
||||
++i) {
|
||||
const std::string &group_id = best_score_in_groups[i].first;
|
||||
group_topk_list.emplace_back(
|
||||
std::make_pair(group_id, group_topk_heaps_[group_id]));
|
||||
}
|
||||
group_results_[idx].resize(group_topk_list.size());
|
||||
for (uint32_t i = 0; i < group_topk_list.size(); ++i) {
|
||||
const std::string &group_id = group_topk_list[i].first;
|
||||
group_results_[idx][i].set_group_id(group_id);
|
||||
uint32_t size = std::min(
|
||||
group_topk_, static_cast<uint32_t>(group_topk_list[i].second.size()));
|
||||
for (uint32_t j = 0; j < size; ++j) {
|
||||
auto score = group_topk_list[i].second[j].second;
|
||||
if (score > this->threshold()) {
|
||||
break;
|
||||
}
|
||||
node_id_t id = group_topk_list[i].second[j].first;
|
||||
auto provider = owner_->create_provider();
|
||||
if (fetch_vector_) {
|
||||
IndexStorage::MemoryBlock block;
|
||||
provider->get_vector(id, block);
|
||||
group_results_[idx][i].mutable_docs()->emplace_back(id, score, id,
|
||||
block);
|
||||
} else {
|
||||
group_results_[idx][i].mutable_docs()->emplace_back(id, score, id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//! Get if group by search
|
||||
bool group_by_search() {
|
||||
return group_num_ > 0;
|
||||
}
|
||||
//! Set group params
|
||||
void set_group_params(uint32_t group_num, uint32_t group_topk) override {
|
||||
group_num_ = group_num;
|
||||
group_topk_ = group_topk;
|
||||
group_topk_heaps_.clear();
|
||||
}
|
||||
|
||||
void reset() override {}
|
||||
|
||||
//! Reset the context
|
||||
void reset(const FlatStreamer<BATCH_SIZE> *owner) {
|
||||
magic_ = owner->magic();
|
||||
feature_size_ = owner->meta().element_size();
|
||||
|
||||
uint32_t block_size = feature_size_ * BATCH_SIZE;
|
||||
actual_read_size_ =
|
||||
(owner->read_block_size() + block_size - 1) / block_size * block_size;
|
||||
owner_ = owner;
|
||||
}
|
||||
|
||||
//! Reset all the query results
|
||||
void reset_results(size_t qnum) {
|
||||
results_.resize(qnum);
|
||||
stats_vec_.resize(qnum);
|
||||
for (size_t i = 0; i < qnum; ++i) {
|
||||
results_[i].clear();
|
||||
stats_vec_[i].clear();
|
||||
}
|
||||
result_heap_.clear();
|
||||
result_heap_.limit(topk_);
|
||||
result_heap_.set_threshold(this->threshold());
|
||||
}
|
||||
|
||||
Stats *mutable_stats(size_t idx = 0) {
|
||||
ailego_assert_with(stats_vec_.size() > idx, "invalid index");
|
||||
return &stats_vec_[idx];
|
||||
}
|
||||
|
||||
private:
|
||||
const FlatStreamer<BATCH_SIZE> *owner_{nullptr};
|
||||
std::vector<Stats> stats_vec_{};
|
||||
uint32_t magic_{0};
|
||||
uint32_t topk_{0};
|
||||
uint32_t feature_size_{0};
|
||||
uint32_t actual_read_size_{0};
|
||||
IndexDocumentHeap result_heap_;
|
||||
std::vector<IndexDocumentList> results_{};
|
||||
std::string batch_queries_{};
|
||||
float scores_[BATCH_SIZE * BATCH_SIZE];
|
||||
bool fetch_vector_{false};
|
||||
// group
|
||||
uint32_t group_num_{0};
|
||||
uint32_t group_topk_{0};
|
||||
std::map<std::string, TopkHeap> group_topk_heaps_{};
|
||||
std::vector<IndexGroupDocumentList> group_results_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,243 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "flat_streamer.h"
|
||||
#include "flat_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
class FlatStreamerDumper {
|
||||
public:
|
||||
typedef std::unique_ptr<FlatStreamerDumper> Pointer;
|
||||
|
||||
FlatStreamerDumper(const FlatStreamer<BATCH_SIZE> *owner) {
|
||||
owner_ = owner;
|
||||
dump_size_ = 0;
|
||||
}
|
||||
|
||||
int dump(const IndexDumper::Pointer &dumper) {
|
||||
ailego::ElapsedTime stamp;
|
||||
|
||||
std::vector<uint64_t> keys;
|
||||
if (owner_->meta().major_order() == IndexMeta::MO_COLUMN) {
|
||||
int error_code = this->write_column_index(dumper.get(), &keys);
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
} else {
|
||||
int error_code = this->write_row_index(dumper.get(), &keys);
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
}
|
||||
|
||||
int error_code = this->write_keys(keys, dumper.get());
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
|
||||
error_code = this->write_mapping(keys, dumper.get());
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
|
||||
error_code = IndexHelper::SerializeToDumper(owner_->meta(), dumper.get());
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
LOG_DEBUG("dumped_count: %zu, costtime: %zu", keys.size(),
|
||||
(size_t)stamp.milli_seconds());
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t dump_size() {
|
||||
return dump_size_;
|
||||
}
|
||||
|
||||
private:
|
||||
int write_column_index(IndexDumper *dumper, std::vector<uint64_t> *keys) {
|
||||
switch (IndexMeta::AlignSizeof(owner_->meta().data_type())) {
|
||||
case 2:
|
||||
return this->write_column_index<uint16_t>(dumper, keys);
|
||||
case 4:
|
||||
return this->write_column_index<uint32_t>(dumper, keys);
|
||||
case 8:
|
||||
return this->write_column_index<uint64_t>(dumper, keys);
|
||||
default:
|
||||
ailego_check_with(0, "BAD CASE");
|
||||
}
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
int write_column_index(IndexDumper *dumper, std::vector<uint64_t> *keys) {
|
||||
auto iter = owner_->entity().creater_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Failed to create iterator");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
// Write features
|
||||
size_t element_size = owner_->meta().element_size();
|
||||
size_t block_size = element_size * BATCH_SIZE;
|
||||
std::string block1, block2;
|
||||
block1.reserve(block_size);
|
||||
block2.reserve(block_size);
|
||||
|
||||
for (; iter->is_valid(); iter->next()) {
|
||||
block1.append(reinterpret_cast<const char *>(iter->data()), element_size);
|
||||
keys->emplace_back(iter->key());
|
||||
|
||||
if (block1.size() == block_size) {
|
||||
ailego::MatrixHelper::Transpose<T, BATCH_SIZE>(
|
||||
block1.data(), element_size / sizeof(T), (void *)block2.data());
|
||||
|
||||
if (dumper->write(block2.data(), block_size) != block_size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s",
|
||||
dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
block1.clear();
|
||||
dump_size_ += block_size;
|
||||
}
|
||||
}
|
||||
|
||||
if (!block1.empty()) {
|
||||
if (dumper->write(block1.data(), block1.size()) != block1.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s",
|
||||
dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
dump_size_ += block1.size();
|
||||
}
|
||||
|
||||
// Write the padding if need
|
||||
size_t features_size = keys->size() * element_size;
|
||||
size_t features_padding_size =
|
||||
ailego_align(features_size, 32) - features_size;
|
||||
if (features_padding_size) {
|
||||
std::string padding(features_padding_size, '\0');
|
||||
|
||||
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s",
|
||||
dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
dump_size_ += padding.size();
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
|
||||
features_padding_size, 0);
|
||||
}
|
||||
|
||||
int write_row_index(IndexDumper *dumper, std::vector<uint64_t> *keys) {
|
||||
auto iter = owner_->entity().creater_iterator();
|
||||
if (!iter) {
|
||||
LOG_ERROR("Failed to create iterator");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
// Write features
|
||||
size_t element_size = owner_->meta().element_size();
|
||||
for (; iter->is_valid(); iter->next()) {
|
||||
if (dumper->write(iter->data(), element_size) != element_size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s",
|
||||
dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
dump_size_ += element_size;
|
||||
keys->emplace_back(iter->key());
|
||||
}
|
||||
|
||||
// Write the padding if need
|
||||
size_t features_size = keys->size() * element_size;
|
||||
size_t features_padding_size =
|
||||
ailego_align(features_size, 32) - features_size;
|
||||
if (features_padding_size) {
|
||||
std::string padding(features_padding_size, '\0');
|
||||
|
||||
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s",
|
||||
dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
dump_size_ += padding.size();
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
|
||||
features_padding_size, 0);
|
||||
}
|
||||
|
||||
int write_keys(const std::vector<uint64_t> &keys, IndexDumper *dumper) {
|
||||
size_t keys_size = keys.size() * sizeof(uint64_t);
|
||||
size_t keys_padding_size = ailego_align(keys_size, 32) - keys_size;
|
||||
if (dumper->write(keys.data(), keys_size) != keys_size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
dump_size_ += keys_size;
|
||||
|
||||
// Write the padding if need
|
||||
if (keys_padding_size) {
|
||||
std::string padding(keys_padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s",
|
||||
dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
dump_size_ += padding.size();
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
|
||||
0);
|
||||
}
|
||||
|
||||
int write_mapping(const std::vector<uint64_t> &keys, IndexDumper *dumper) {
|
||||
std::vector<uint32_t> mapping(keys.size());
|
||||
std::iota(mapping.begin(), mapping.end(), 0);
|
||||
std::sort(mapping.begin(), mapping.end(),
|
||||
[&keys](uint32_t lhs, uint32_t rhs) {
|
||||
return (keys[lhs] < keys[rhs]);
|
||||
});
|
||||
|
||||
size_t mapping_size = mapping.size() * sizeof(uint32_t);
|
||||
size_t mapping_padding_size = ailego_align(mapping_size, 32) - mapping_size;
|
||||
if (dumper->write(mapping.data(), mapping_size) != mapping_size) {
|
||||
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
dump_size_ += mapping_size;
|
||||
|
||||
// Write the padding if need
|
||||
if (mapping_padding_size) {
|
||||
std::string padding(mapping_padding_size, '\0');
|
||||
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
|
||||
LOG_ERROR("Failed to write data into dumper %s",
|
||||
dumper->name().c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
dump_size_ += padding.size();
|
||||
}
|
||||
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
|
||||
mapping_padding_size, 0);
|
||||
}
|
||||
|
||||
private:
|
||||
const FlatStreamer<BATCH_SIZE> *owner_{nullptr};
|
||||
size_t dump_size_{0};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,994 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "flat_streamer_entity.h"
|
||||
#include <cstdint>
|
||||
#include "flat_utility.h"
|
||||
#include "index_error.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
FlatStreamerEntity::FlatStreamerEntity(IndexStreamer::Stats &stats)
|
||||
: stats_(stats) {}
|
||||
|
||||
int FlatStreamerEntity::open(IndexStorage::Pointer storage,
|
||||
const IndexMeta & /*mt*/) {
|
||||
if (storage_) {
|
||||
LOG_ERROR("An storage instance is already opened");
|
||||
return IndexError_Duplicate;
|
||||
}
|
||||
// segments_[0] store the meta information of the linear list
|
||||
ailego_assert_with(segments_.size() == 0, "Invalid Size");
|
||||
|
||||
key_info_map_lock_ = std::make_shared<ailego::SharedMutex>();
|
||||
key_info_map_.clear();
|
||||
id_key_vector_.clear();
|
||||
withid_key_info_map_.clear();
|
||||
withid_key_map_.clear();
|
||||
|
||||
vec_unit_size_ = IndexMeta::AlignSizeof(index_meta_.data_type());
|
||||
vec_cols_ = index_meta_.element_size() / vec_unit_size_;
|
||||
meta_.header.block_size =
|
||||
ailego_align(sizeof(BlockHeader) + sizeof(DeletionMap) +
|
||||
(index_meta_.element_size() + sizeof(uint64_t)) *
|
||||
meta_.header.block_vector_count,
|
||||
32);
|
||||
|
||||
if (storage->get(FLAT_LINEAR_LIST_HEAD_SEG_ID) ||
|
||||
storage->get(FLAT_LINEAR_META_SEG_ID)) {
|
||||
int ret = this->load_storage(storage);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Failed to load storage index");
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
int ret = this->init_storage(storage);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Failed to init storage");
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
storage_ = storage;
|
||||
|
||||
//! Create the distance calculator
|
||||
auto metric = IndexFactory::CreateMetric(index_meta_.metric_name());
|
||||
if (!metric) {
|
||||
LOG_ERROR("Failed to create metric %s", index_meta_.metric_name().c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
int ret = metric->init(index_meta_, index_meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize metric %s",
|
||||
index_meta_.metric_name().c_str());
|
||||
return ret;
|
||||
}
|
||||
row_distance_ = metric->distance();
|
||||
column_distance_ =
|
||||
metric->distance_matrix(meta_.header.block_vector_count, 1);
|
||||
|
||||
LOG_DEBUG("Open storage %s done, metric=%s", storage_->name().c_str(),
|
||||
index_meta_.metric_name().c_str());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::close(void) {
|
||||
segments_.clear();
|
||||
storage_.reset();
|
||||
key_info_map_lock_.reset();
|
||||
key_info_map_.clear();
|
||||
withid_key_info_map_.clear();
|
||||
withid_key_map_.clear();
|
||||
id_key_vector_.clear();
|
||||
meta_.create_time = 0;
|
||||
meta_.update_time = 0;
|
||||
meta_.segment_count = 0;
|
||||
meta_.header.total_vector_count = 0;
|
||||
meta_.header.block_count = 0;
|
||||
meta_.header.block_size = 0;
|
||||
meta_.header.linear_body_size = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::flush_linear_meta(void) {
|
||||
if (!storage_) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
meta_.update_time = ailego::Realtime::Seconds();
|
||||
meta_.revision_id = stats_.revision_id();
|
||||
stats_.set_update_time(meta_.update_time);
|
||||
auto segment = storage_->get(FLAT_LINEAR_META_SEG_ID);
|
||||
if (ailego_unlikely(!segment)) {
|
||||
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
if (segment->write(0, &meta_, sizeof(meta_)) != sizeof(meta_)) {
|
||||
LOG_ERROR("Failed to write segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::flush(uint64_t checkpoint) {
|
||||
int ret = this->flush_linear_meta();
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (checkpoint != 0) {
|
||||
storage_->refresh(checkpoint);
|
||||
}
|
||||
ret = storage_->flush();
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Failed to refresh storage for %s", IndexError::What(ret));
|
||||
return ret;
|
||||
}
|
||||
if (checkpoint != 0) {
|
||||
stats_.set_check_point(checkpoint);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::add(uint64_t key, const void *vec, size_t size) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (filter_same_key_) {
|
||||
key_info_map_lock_->lock_shared();
|
||||
if (key_info_map_.find(key) != key_info_map_.end()) {
|
||||
key_info_map_lock_->unlock_shared();
|
||||
LOG_WARN("Try to add duplicate key, drop it");
|
||||
return IndexError_Duplicate;
|
||||
}
|
||||
key_info_map_lock_->unlock_shared();
|
||||
}
|
||||
if (size != static_cast<size_t>(index_meta_.element_size())) {
|
||||
LOG_ERROR("Failed to add, mismatch size %zu vs elemsize %u", size,
|
||||
index_meta_.element_size());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
IndexStorage::MemoryBlock head_block;
|
||||
this->get_head_block(head_block);
|
||||
const BlockLocation *bl =
|
||||
reinterpret_cast<const BlockLocation *>(head_block.data());
|
||||
if (ailego_unlikely(bl == nullptr)) {
|
||||
LOG_ERROR("Failed to get block loc");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
BlockLocation block = *bl;
|
||||
|
||||
if (!this->is_valid_block(block)) {
|
||||
int ret = this->alloc_block(block, &block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
ret = this->update_head_block(block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
int ret = this->add_to_block(block, key, vec, size);
|
||||
if (ret == IndexError_IndexFull) {
|
||||
ret = this->alloc_block(block, &block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
ret = this->update_head_block(block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
ret = this->add_to_block(block, key, vec, size);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
(*stats_.mutable_added_count())++;
|
||||
stats_.set_revision_id(meta_.revision_id + 1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::search(const void *query, const IndexFilter &filter,
|
||||
uint32_t *scan_count, IndexDocumentHeap *heap,
|
||||
IndexContext::Stats *context_stats) const {
|
||||
IndexStorage::MemoryBlock head_block;
|
||||
this->get_head_block(head_block);
|
||||
const BlockLocation *bl =
|
||||
reinterpret_cast<const BlockLocation *>(head_block.data());
|
||||
if (ailego_unlikely(bl == nullptr)) {
|
||||
LOG_ERROR("Failed to get block loc");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
BlockLocation block = *bl;
|
||||
|
||||
while (this->is_valid_block(block)) {
|
||||
IndexStorage::MemoryBlock block_header_block;
|
||||
this->get_block_header(block, block_header_block);
|
||||
const BlockHeader *hd =
|
||||
reinterpret_cast<const BlockHeader *>(block_header_block.data());
|
||||
if (ailego_unlikely(hd == nullptr)) {
|
||||
LOG_ERROR("Failed to get block header");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
if (hd->vector_count > 0) {
|
||||
*scan_count += hd->vector_count;
|
||||
IndexStorage::MemoryBlock deletion_map_block;
|
||||
this->get_block_deletion_map(block, deletion_map_block);
|
||||
const DeletionMap *deletion_map =
|
||||
reinterpret_cast<const DeletionMap *>(deletion_map_block.data());
|
||||
if (filter.is_valid() || deletion_map->is_dirty()) {
|
||||
this->search_block(query, block, hd, 1.0, filter, deletion_map, heap,
|
||||
context_stats);
|
||||
} else {
|
||||
*(context_stats->mutable_dist_calced_count()) += hd->vector_count;
|
||||
this->search_block(query, block, hd, 1.0, heap);
|
||||
}
|
||||
}
|
||||
block = hd->next;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Search in a block
|
||||
void FlatStreamerEntity::search_block(const void *query,
|
||||
const BlockLocation &bl,
|
||||
const BlockHeader *hd, float norm_val,
|
||||
IndexDocumentHeap *heap) const {
|
||||
std::vector<float> distances(block_vector_count());
|
||||
IndexStorage::MemoryBlock vecs_block;
|
||||
this->get_block_vectors(bl, vecs_block);
|
||||
const char *vecs = reinterpret_cast<const char *>(vecs_block.data());
|
||||
IndexStorage::MemoryBlock keys_block;
|
||||
this->get_block_keys(bl, keys_block);
|
||||
const uint64_t *keys = reinterpret_cast<const uint64_t *>(keys_block.data());
|
||||
row_major_distance(query, vecs, hd->vector_count, distances.data());
|
||||
for (size_t k = 0; k < hd->vector_count; ++k) {
|
||||
if (keys[k] != kInvalidKey) {
|
||||
heap->emplace(keys[k], distances[k] * norm_val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//! Search in a block with filter
|
||||
void FlatStreamerEntity::search_block(
|
||||
const void *query, const BlockLocation &bl, const BlockHeader *hd,
|
||||
float norm_val, const IndexFilter &filter, const DeletionMap *deletion_map,
|
||||
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const {
|
||||
std::vector<float> distances(block_vector_count());
|
||||
|
||||
IndexStorage::MemoryBlock vecs_block;
|
||||
this->get_block_vectors(bl, vecs_block);
|
||||
const char *vecs = reinterpret_cast<const char *>(vecs_block.data());
|
||||
IndexStorage::MemoryBlock keys_block;
|
||||
this->get_block_keys(bl, keys_block);
|
||||
const uint64_t *keys = reinterpret_cast<const uint64_t *>(keys_block.data());
|
||||
|
||||
DeletionMap keeps;
|
||||
for (size_t k = 0; k < hd->vector_count; ++k) {
|
||||
const bool condition1 = !deletion_map->test(k);
|
||||
const bool condition2 = filter.is_valid() ? !filter(keys[k]) : true;
|
||||
const bool condition3 = keys[k] != kInvalidKey;
|
||||
if (condition1 && condition2 && condition3) {
|
||||
keeps.set(k);
|
||||
}
|
||||
}
|
||||
if (!keeps.is_dirty()) {
|
||||
(*context_stats->mutable_filtered_count()) += hd->vector_count;
|
||||
return;
|
||||
}
|
||||
for (size_t k = 0; k < hd->vector_count; ++k) {
|
||||
if (keeps.test(k)) {
|
||||
auto cur_vec = vecs + index_meta_.element_size() * k;
|
||||
row_major_distance(query, cur_vec, 1, distances.data() + k);
|
||||
++(*context_stats->mutable_dist_calced_count());
|
||||
}
|
||||
}
|
||||
for (size_t k = 0; k < hd->vector_count; ++k) {
|
||||
if (keeps.test(k)) {
|
||||
heap->emplace(keys[k], distances[k] * norm_val);
|
||||
} else {
|
||||
++(*context_stats->mutable_filtered_count());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::search_bf(const void *query, const IndexFilter &filter,
|
||||
IndexDocumentHeap *heap,
|
||||
IndexContext::Stats *context_stats) const {
|
||||
uint32_t scan_count;
|
||||
return this->search(query, filter, &scan_count, heap, context_stats);
|
||||
}
|
||||
|
||||
FlatStreamerEntity::Pointer FlatStreamerEntity::clone(void) const {
|
||||
std::vector<IndexStorage::Segment::Pointer> segments;
|
||||
segments.reserve(segments_.size());
|
||||
for (size_t i = 0; i < segments_.size(); ++i) {
|
||||
segments.emplace_back(segments_[i]->clone());
|
||||
if (!segments[i]) {
|
||||
LOG_ERROR("Failed to clone segment, index=%zu", i);
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
auto entity = new (std::nothrow) FlatStreamerEntity(stats_);
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to New FlatStreamerEntity object");
|
||||
return nullptr;
|
||||
}
|
||||
entity->index_meta_ = this->index_meta_;
|
||||
entity->storage_ = this->storage_;
|
||||
// entity->reformer_ = this->reformer_;
|
||||
entity->segments_ = segments;
|
||||
entity->meta_ = this->meta_;
|
||||
entity->key_info_map_lock_ = this->key_info_map_lock_;
|
||||
entity->key_info_map_ = this->key_info_map_;
|
||||
entity->id_key_vector_ = this->id_key_vector_;
|
||||
entity->withid_key_info_map_ = this->withid_key_info_map_;
|
||||
entity->withid_key_map_ = this->withid_key_map_;
|
||||
entity->filter_same_key_ = this->filter_same_key_;
|
||||
entity->vec_unit_size_ = this->vec_unit_size_;
|
||||
entity->vec_cols_ = this->vec_cols_;
|
||||
return FlatStreamerEntity::Pointer(entity);
|
||||
}
|
||||
|
||||
const void *FlatStreamerEntity::get_vector_by_key(uint64_t key) const {
|
||||
VectorLocation loc{};
|
||||
key_info_map_lock_->lock_shared();
|
||||
if (use_key_info_map_) {
|
||||
auto iterator = key_info_map_.find(key);
|
||||
if (iterator == key_info_map_.end()) {
|
||||
key_info_map_lock_->unlock_shared();
|
||||
return nullptr;
|
||||
}
|
||||
loc = iterator->second;
|
||||
} else {
|
||||
if (key < withid_key_info_map_.size()) {
|
||||
loc = withid_key_info_map_[key];
|
||||
} else {
|
||||
key_info_map_lock_->unlock_shared();
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
key_info_map_lock_->unlock_shared();
|
||||
|
||||
auto segment = this->get_segment(loc.segment_id);
|
||||
const void *data = nullptr;
|
||||
if (segment->read(loc.offset, &data, index_meta_.element_size()) !=
|
||||
index_meta_.element_size()) {
|
||||
LOG_ERROR("Failed to read segment, size=%u", index_meta_.element_size());
|
||||
return nullptr;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::get_vector_by_key(
|
||||
const uint64_t key, IndexStorage::MemoryBlock &block) const {
|
||||
VectorLocation loc{};
|
||||
key_info_map_lock_->lock_shared();
|
||||
if (use_key_info_map_) {
|
||||
auto iterator = key_info_map_.find(key);
|
||||
if (iterator == key_info_map_.end()) {
|
||||
key_info_map_lock_->unlock_shared();
|
||||
return -1;
|
||||
}
|
||||
loc = iterator->second;
|
||||
} else {
|
||||
if (key < withid_key_info_map_.size()) {
|
||||
loc = withid_key_info_map_[key];
|
||||
} else {
|
||||
key_info_map_lock_->unlock_shared();
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
key_info_map_lock_->unlock_shared();
|
||||
|
||||
auto segment = this->get_segment(loc.segment_id);
|
||||
if (segment->read(loc.offset, block, index_meta_.element_size()) !=
|
||||
index_meta_.element_size()) {
|
||||
LOG_ERROR("Failed to read segment, size=%u", index_meta_.element_size());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
IndexProvider::Iterator::Pointer FlatStreamerEntity::creater_iterator(
|
||||
void) const {
|
||||
auto entity = this->clone();
|
||||
if (!entity) {
|
||||
LOG_ERROR("Failed to clone entity");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return Iterator::Pointer(new (std::nothrow)
|
||||
FlatStreamerEntity::Iterator(std::move(entity)));
|
||||
}
|
||||
|
||||
void FlatStreamerEntity::Iterator::read_next_block(void) {
|
||||
auto block_size = entity_->linear_block_size();
|
||||
while (segment_id_ < entity_->segments_.size()) {
|
||||
auto &segment = entity_->segments_[segment_id_];
|
||||
size_t off = block_index_ * block_size;
|
||||
if (off + block_size > segment->data_size()) {
|
||||
++segment_id_;
|
||||
block_index_ = 0;
|
||||
continue;
|
||||
}
|
||||
if (segment->read(off, block_, block_size) != block_size) {
|
||||
LOG_ERROR("Failed to read block, off=%zu", off);
|
||||
break;
|
||||
}
|
||||
data_ = block_.data();
|
||||
auto hd = reinterpret_cast<const BlockHeader *>(
|
||||
static_cast<const char *>(data_) + block_size - sizeof(BlockHeader));
|
||||
if (hd->vector_count == 0) {
|
||||
++block_index_;
|
||||
continue;
|
||||
}
|
||||
|
||||
block_vector_count_ = hd->vector_count;
|
||||
block_vector_index_ = 0;
|
||||
size_t elemsize = entity_->index_meta_.element_size();
|
||||
keys_ = reinterpret_cast<const uint64_t *>(
|
||||
reinterpret_cast<const char *>(data_) +
|
||||
elemsize * entity_->block_vector_count());
|
||||
return;
|
||||
}
|
||||
|
||||
is_valid_ = false;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::init_storage(IndexStorage::Pointer storage) {
|
||||
// Init Linear Meta Segment
|
||||
meta_.create_time = ailego::Realtime::Seconds();
|
||||
stats_.set_create_time(meta_.create_time);
|
||||
meta_.update_time = ailego::Realtime::Seconds();
|
||||
stats_.set_update_time(meta_.update_time);
|
||||
meta_.segment_count = 0;
|
||||
meta_.revision_id = 0;
|
||||
|
||||
std::string str;
|
||||
index_meta_.serialize(&str);
|
||||
const size_t page = ailego::MemoryHelper::PageSize();
|
||||
|
||||
meta_.header.header_size = sizeof(LinearIndexHeader) + str.size();
|
||||
meta_.header.total_vector_count = 0;
|
||||
meta_.header.linear_body_size = 0;
|
||||
meta_.header.block_count = 0;
|
||||
meta_.header.index_meta_size = str.size();
|
||||
meta_.header.linear_list_count = 1;
|
||||
|
||||
AdjustSegmentSize(&meta_);
|
||||
|
||||
LOG_DEBUG(
|
||||
"Create Streamer Index, VecSize=%u, BlockSize=%u SegmentSize=%u "
|
||||
"LinearListCount=%u",
|
||||
index_meta_.element_size(), meta_.header.block_size, meta_.segment_size,
|
||||
meta_.header.linear_list_count);
|
||||
|
||||
size_t size = ailego_align(sizeof(meta_) + str.size(), page);
|
||||
int ret = storage->append(FLAT_LINEAR_META_SEG_ID, size);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Failed to append segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
|
||||
return ret;
|
||||
}
|
||||
auto segment = storage->get(FLAT_LINEAR_META_SEG_ID);
|
||||
if (ailego_unlikely(!segment)) {
|
||||
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
if (segment->write(0, &meta_, sizeof(meta_)) != sizeof(meta_)) {
|
||||
LOG_ERROR("Failed to write segment data");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
if (segment->write(sizeof(meta_), str.data(), str.size()) != str.size()) {
|
||||
LOG_ERROR("Failed to write segment data, size=%zu", str.size());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
ret = storage->append("IndexMeta", str.size());
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Failed to append segment IndexMeta, code: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
auto index_meta_segment = storage->get("IndexMeta");
|
||||
if (index_meta_segment->write(0, str.data(), str.size()) != str.size()) {
|
||||
LOG_ERROR("Failed to write segment data, size=%zu", str.size());
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
*stats_.mutable_index_size() += size;
|
||||
|
||||
// Init Linear List Head Segment
|
||||
size = ailego_align(sizeof(BlockLocation) * linear_list_count(), page);
|
||||
ret = storage->append(FLAT_LINEAR_LIST_HEAD_SEG_ID, size);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Failed to append segment %s for %s, size=%zu",
|
||||
FLAT_LINEAR_LIST_HEAD_SEG_ID.c_str(), IndexError::What(ret),
|
||||
size);
|
||||
return ret;
|
||||
}
|
||||
segment = storage->get(FLAT_LINEAR_LIST_HEAD_SEG_ID);
|
||||
if (ailego_unlikely(!segment)) {
|
||||
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_LIST_HEAD_SEG_ID.c_str());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
if (segment->resize(size) != size) {
|
||||
LOG_ERROR("Failed to resize segment, size=%zu", size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
segments_.emplace_back(std::move(segment));
|
||||
|
||||
*stats_.mutable_index_size() += size;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::load_linear_meta(IndexStorage::Pointer storage) {
|
||||
AdjustSegmentSize(&meta_);
|
||||
|
||||
// Load Meta Segment
|
||||
auto segment = storage->get(FLAT_LINEAR_META_SEG_ID);
|
||||
if (!segment || segment->data_size() < sizeof(meta_)) {
|
||||
LOG_ERROR("Missing segment %s, or invalid segment size",
|
||||
FLAT_LINEAR_META_SEG_ID.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
IndexStorage::MemoryBlock data_block;
|
||||
if (segment->read(0, data_block, segment->data_size()) !=
|
||||
segment->data_size()) {
|
||||
LOG_ERROR("Failed to read storage, size=%zu", segment->data_size());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
auto *mt = reinterpret_cast<const decltype(meta_) *>(data_block.data());
|
||||
if (mt->header.block_vector_count != meta_.header.block_vector_count) {
|
||||
LOG_ERROR("Unmatched BlockVecCount Setting, Index %u vs Setting %u",
|
||||
mt->header.block_vector_count, meta_.header.block_vector_count);
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
if (mt->header.block_size != meta_.header.block_size) {
|
||||
LOG_ERROR("Unmatched BlockSize Setting, Index %u vs Setting %u",
|
||||
mt->header.block_size, meta_.header.block_size);
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
if (mt->header.index_meta_size + sizeof(meta_) > segment->data_size()) {
|
||||
LOG_ERROR("Invalid format, IndexMetaSize %u, SegmentSize %zu",
|
||||
mt->header.index_meta_size, segment->data_size());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (mt->header.linear_list_count != meta_.header.linear_list_count) {
|
||||
LOG_ERROR("Unmatch LinearListCount, Index size %u vs Setting %u",
|
||||
mt->header.linear_list_count, meta_.header.linear_list_count);
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
IndexMeta index_meta;
|
||||
if (!index_meta.deserialize(mt->header.index_meta,
|
||||
mt->header.index_meta_size)) {
|
||||
LOG_ERROR("Failed to deserialize IndexMeta, size=%u",
|
||||
mt->header.index_meta_size);
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
if (index_meta.data_type() != index_meta_.data_type() ||
|
||||
index_meta.dimension() != index_meta_.dimension() ||
|
||||
index_meta.element_size() != index_meta_.element_size() ||
|
||||
index_meta.metric_name() != index_meta_.metric_name()) {
|
||||
LOG_ERROR(
|
||||
"Unmatch IndexMeta, Index(type=%u dim=%u elemsize=%u "
|
||||
"metric=%s) Setting(type=%u dim=%u elemsize=%u metric=%s)",
|
||||
index_meta.data_type(), index_meta.dimension(),
|
||||
index_meta.element_size(), index_meta.metric_name().c_str(),
|
||||
index_meta_.data_type(), index_meta_.dimension(),
|
||||
index_meta_.element_size(), index_meta_.metric_name().c_str());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
// Segment Size can be reconfigurable
|
||||
auto segment_size = meta_.segment_size;
|
||||
std::memcpy(&meta_, mt, sizeof(meta_));
|
||||
meta_.segment_size = segment_size;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::load_segment_keys_to_map(BlockLocation block) {
|
||||
while (this->is_valid_block(block)) {
|
||||
auto segment = this->get_segment(block.segment_id);
|
||||
|
||||
IndexStorage::MemoryBlock block_header_block;
|
||||
this->get_block_header(block, block_header_block);
|
||||
const BlockHeader *hd =
|
||||
reinterpret_cast<const BlockHeader *>(block_header_block.data());
|
||||
if (ailego_unlikely(hd == nullptr)) {
|
||||
LOG_ERROR("Failed to get block header");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
IndexStorage::MemoryBlock keys_block;
|
||||
this->get_block_keys(block, keys_block);
|
||||
const uint64_t *keys =
|
||||
reinterpret_cast<const uint64_t *>(keys_block.data());
|
||||
IndexStorage::MemoryBlock deletion_map_block;
|
||||
this->get_block_deletion_map(block, deletion_map_block);
|
||||
const DeletionMap *deletion_map =
|
||||
reinterpret_cast<const DeletionMap *>(deletion_map_block.data());
|
||||
|
||||
for (uint32_t vector_index = 0; vector_index < hd->vector_count;
|
||||
++vector_index) {
|
||||
if (deletion_map->test(vector_index)) {
|
||||
continue;
|
||||
}
|
||||
size_t vector_off =
|
||||
this->get_block_vector_offset(block.block_index, vector_index);
|
||||
key_info_map_[keys[vector_index]] =
|
||||
VectorLocation(block.segment_id, false, vector_off);
|
||||
id_key_vector_.push_back(keys[vector_index]);
|
||||
}
|
||||
block = hd->next;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::load_segment_keys_to_vector() {
|
||||
for (uint32_t i = 0; i < meta_.header.total_vector_count; i++) {
|
||||
size_t block_id = i / block_vector_count();
|
||||
uint32_t vector_index = i % block_vector_count();
|
||||
|
||||
ailego_assert(segments_.size() > 1);
|
||||
size_t segment_block_count =
|
||||
segments_[1]->data_size() / linear_block_size();
|
||||
size_t segment_id = block_id / segment_block_count + 1;
|
||||
size_t real_block_id = block_id % segment_block_count;
|
||||
size_t vector_off =
|
||||
this->get_block_vector_offset(real_block_id, vector_index);
|
||||
|
||||
withid_key_info_map_.push_back(
|
||||
VectorLocation(segment_id, false, vector_off));
|
||||
size_t key_off = get_block_key_offset(real_block_id, vector_index);
|
||||
withid_key_map_.push_back(key_off);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::load_storage(IndexStorage::Pointer storage) {
|
||||
int ret = this->load_linear_meta(storage);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Load Linear List
|
||||
auto hd_segment = storage->get(FLAT_LINEAR_LIST_HEAD_SEG_ID);
|
||||
if (ailego_unlikely(!hd_segment)) {
|
||||
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_LIST_HEAD_SEG_ID.c_str());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
if (hd_segment->data_size() < linear_list_count() * sizeof(BlockLocation)) {
|
||||
LOG_ERROR("Invalid segment size, LinearListCount=%zu, size=%zu",
|
||||
linear_list_count(), hd_segment->data_size());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
segments_.emplace_back(hd_segment);
|
||||
|
||||
size_t index_size = hd_segment->capacity();
|
||||
for (size_t i = 1; i <= meta_.segment_count; ++i) {
|
||||
std::string segment_id =
|
||||
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
|
||||
auto seg = storage->get(segment_id);
|
||||
if (!seg || seg->data_size() < meta_.header.block_size) {
|
||||
LOG_ERROR("Failed to get segment %s, or invalid segment size",
|
||||
segment_id.c_str());
|
||||
return IndexError_InvalidFormat;
|
||||
}
|
||||
index_size += seg->capacity();
|
||||
segments_.emplace_back(std::move(seg));
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < linear_list_count(); i++) {
|
||||
IndexStorage::MemoryBlock head_block;
|
||||
this->get_head_block(head_block);
|
||||
const BlockLocation *bl =
|
||||
reinterpret_cast<const BlockLocation *>(head_block.data());
|
||||
if (ailego_unlikely(bl == nullptr)) {
|
||||
LOG_ERROR("Failed to get block loc");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
BlockLocation block = *bl;
|
||||
if (use_key_info_map_) {
|
||||
ret = this->load_segment_keys_to_map(block);
|
||||
} else {
|
||||
ret = this->load_segment_keys_to_vector();
|
||||
}
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
char create_time[32];
|
||||
char update_time[32];
|
||||
ailego::Realtime::Gmtime(meta_.create_time, "%Y-%m-%d %H:%M:%S", create_time,
|
||||
sizeof(create_time));
|
||||
ailego::Realtime::Gmtime(meta_.update_time, "%Y-%m-%d %H:%M:%S", update_time,
|
||||
sizeof(update_time));
|
||||
LOG_DEBUG(
|
||||
"Load Index, IndexSize=%zu SegmentCount=%u SegmentSize=%u "
|
||||
"RevisionId=%zu BlockCount=%u BlockSize=%u "
|
||||
"BlockVectorCount=%u LinearListCount=%u TotalVecCount=%zu "
|
||||
"CreateTime=%s UpdateTime=%s",
|
||||
index_size, meta_.segment_count, meta_.segment_size,
|
||||
static_cast<size_t>(meta_.revision_id), meta_.header.block_count,
|
||||
meta_.header.block_size, meta_.header.block_vector_count,
|
||||
meta_.header.linear_list_count,
|
||||
static_cast<size_t>(meta_.header.total_vector_count), create_time,
|
||||
update_time);
|
||||
|
||||
stats_.set_index_size(index_size);
|
||||
stats_.set_check_point(storage->check_point());
|
||||
stats_.set_create_time(meta_.create_time);
|
||||
stats_.set_revision_id(meta_.revision_id);
|
||||
stats_.set_update_time(meta_.update_time);
|
||||
stats_.set_loaded_count(meta_.header.total_vector_count);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::alloc_segment(void) {
|
||||
size_t index = segments_.size();
|
||||
if (index == kMaxSegmentId) {
|
||||
LOG_ERROR("Failed to alloc new segment, exceed max count %zu",
|
||||
kMaxSegmentId);
|
||||
return IndexError_IndexFull;
|
||||
}
|
||||
|
||||
std::string segment_id =
|
||||
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, index);
|
||||
size_t size =
|
||||
ailego_align(meta_.segment_size, ailego::MemoryHelper::PageSize());
|
||||
auto segment = storage_->get(segment_id);
|
||||
if (segment) {
|
||||
if (segment->padding_size() < linear_block_size()) {
|
||||
LOG_ERROR(
|
||||
"Unexpect segment, index=%zu, data_size=%zu "
|
||||
"padding_size=%zu block_size=%zu",
|
||||
index, segment->data_size(), segment->padding_size(),
|
||||
linear_block_size());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
LOG_WARN("Alloc an existing segment=%s capacity=%zu", segment_id.c_str(),
|
||||
segment->capacity());
|
||||
} else {
|
||||
int ret = storage_->append(segment_id, size);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
LOG_ERROR("Failed to alloc segment from storage");
|
||||
return ret;
|
||||
}
|
||||
segment = storage_->get(segment_id);
|
||||
if (ailego_unlikely(!segment)) {
|
||||
LOG_ERROR("Failed to get segment %s", segment_id.c_str());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
}
|
||||
meta_.segment_count += 1;
|
||||
meta_.header.linear_body_size += size;
|
||||
segments_.emplace_back(std::move(segment));
|
||||
*stats_.mutable_index_size() += size;
|
||||
|
||||
// Update meta information
|
||||
auto meta_segment = storage_->get(FLAT_LINEAR_META_SEG_ID);
|
||||
if (ailego_unlikely(!meta_segment)) {
|
||||
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
if (meta_segment->write(0, &meta_, sizeof(meta_)) != sizeof(meta_)) {
|
||||
LOG_ERROR("Failed to write meta segment");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::alloc_block(const BlockLocation &next,
|
||||
BlockLocation *block) {
|
||||
if (segments_.size() <= 1 ||
|
||||
segments_.back()->padding_size() < linear_block_size()) {
|
||||
int ret = this->alloc_segment();
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
auto &segment = segments_.back();
|
||||
size_t block_index = segment->data_size() / linear_block_size();
|
||||
if (block_index == kMaxBlockId) {
|
||||
LOG_ERROR("Failed to alloc block, exceed max count %zu per segment",
|
||||
kMaxBlockId);
|
||||
return IndexError_IndexFull;
|
||||
}
|
||||
|
||||
BlockHeader header;
|
||||
header.next = next;
|
||||
header.vector_count = 0;
|
||||
header.column_major = false;
|
||||
|
||||
size_t hd_off = segment->data_size() + linear_block_size() - sizeof(header);
|
||||
if (segment->write(hd_off, &header, sizeof(header)) != sizeof(header)) {
|
||||
LOG_ERROR("Failed to write block header");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
size_t del_off = hd_off - sizeof(DeletionMap);
|
||||
DeletionMap reset_del_map{};
|
||||
if (segment->write(del_off, &reset_del_map, sizeof(reset_del_map)) !=
|
||||
sizeof(reset_del_map)) {
|
||||
LOG_ERROR("Failed to write block deletion map");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
++meta_.header.block_count;
|
||||
block->segment_id = segments_.size() - 1;
|
||||
block->block_index = (segment->data_size() / linear_block_size()) - 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::add_to_block(const BlockLocation &block, uint64_t key,
|
||||
const void *data, size_t size) {
|
||||
IndexStorage::MemoryBlock block_header_block;
|
||||
this->get_block_header(block, block_header_block);
|
||||
const BlockHeader *header =
|
||||
reinterpret_cast<const BlockHeader *>(block_header_block.data());
|
||||
if (ailego_unlikely(header == nullptr)) {
|
||||
LOG_ERROR("Failed to get header");
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
|
||||
if (header->vector_count == block_vector_count()) {
|
||||
return IndexError_IndexFull;
|
||||
}
|
||||
|
||||
auto &segment = segments_[block.segment_id];
|
||||
|
||||
size_t vector_off =
|
||||
get_block_vector_offset(block.block_index, header->vector_count);
|
||||
if (segment->write(vector_off, data, size) != size) {
|
||||
LOG_ERROR("Failed to write vector, off=%zu size=%zu", vector_off, size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
size_t key_off =
|
||||
get_block_key_offset(block.block_index, header->vector_count);
|
||||
if (segment->write(key_off, &key, sizeof(key)) != sizeof(key)) {
|
||||
LOG_ERROR("Failed to write key, off=%zu", key_off);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
BlockHeader hd = *header;
|
||||
hd.vector_count += 1;
|
||||
size_t hd_off = get_block_header_offset(block.block_index);
|
||||
if (segment->write(hd_off, &hd, sizeof(hd)) != sizeof(hd)) {
|
||||
LOG_ERROR("Failed to write block header, off=%zu", hd_off);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
VectorLocation loc(block.segment_id, false, vector_off);
|
||||
key_info_map_lock_->lock();
|
||||
key_info_map_[key] = loc;
|
||||
id_key_vector_.push_back(key);
|
||||
withid_key_info_map_.push_back(loc);
|
||||
withid_key_map_.push_back(key_off);
|
||||
key_info_map_lock_->unlock();
|
||||
|
||||
++meta_.header.total_vector_count;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FlatStreamerEntity::add_vector_with_id(const uint32_t id, const void *query,
|
||||
const uint32_t size) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
// if (filter_same_key_) {
|
||||
// key_info_map_lock_->lock_shared();
|
||||
// if (key_info_map_.find(id) != key_info_map_.end()) {
|
||||
// key_info_map_lock_->unlock_shared();
|
||||
// LOG_WARN("Try to add duplicate key, drop it");
|
||||
// return IndexError_Duplicate;
|
||||
// }
|
||||
// key_info_map_lock_->unlock_shared();
|
||||
// }
|
||||
|
||||
if (size != static_cast<size_t>(index_meta_.element_size())) {
|
||||
LOG_ERROR("Failed to add, mismatch size %u vs elemsize %u", size,
|
||||
index_meta_.element_size());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
|
||||
if (id >= vector_count()) {
|
||||
IndexStorage::MemoryBlock head_block;
|
||||
this->get_head_block(head_block);
|
||||
BlockLocation block =
|
||||
*reinterpret_cast<const BlockLocation *>(head_block.data());
|
||||
if (!this->is_valid_block(block)) {
|
||||
int ret = this->alloc_block(block, &block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
ret = this->update_head_block(block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
for (size_t start_id = vector_count(); start_id < id; ++start_id) {
|
||||
std::vector<char> vec(size);
|
||||
int ret = this->add_to_block(block, kInvalidKey, vec.data(), size);
|
||||
if (ret == IndexError_IndexFull) {
|
||||
ret = this->alloc_block(block, &block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
ret = this->update_head_block(block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
ret = this->add_to_block(block, kInvalidKey, vec.data(), size);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int ret = this->add_to_block(block, id, query, size);
|
||||
if (ret == IndexError_IndexFull) {
|
||||
ret = this->alloc_block(block, &block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
ret = this->update_head_block(block);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
ret = this->add_to_block(block, id, query, size);
|
||||
if (ailego_unlikely(ret != 0)) {
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
VectorLocation vector_loc = withid_key_info_map_[id];
|
||||
auto segment = this->get_segment(vector_loc.segment_id);
|
||||
size_t vector_off = vector_loc.offset;
|
||||
if (segment->write(vector_off, query, size) != size) {
|
||||
LOG_ERROR("Failed to write vector, off=%zu size=%u", vector_off, size);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
size_t key_off = withid_key_map_[id];
|
||||
uint64_t key = id;
|
||||
if (segment->write(key_off, &key, sizeof(key)) != sizeof(key)) {
|
||||
LOG_ERROR("Failed to write key, off=%zu", key_off);
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
key_info_map_lock_->lock();
|
||||
key_info_map_[key] = vector_loc;
|
||||
key_info_map_lock_->unlock();
|
||||
}
|
||||
(*stats_.mutable_added_count())++;
|
||||
stats_.set_revision_id(meta_.revision_id + 1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,407 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <unordered_map>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <ailego/utility/memory_helper.h>
|
||||
#include <ailego/utility/string_helper.h>
|
||||
#include <framework/index_context.h>
|
||||
#include <framework/index_framework.h>
|
||||
#include <framework/index_streamer.h>
|
||||
#include "flat_index_format.h"
|
||||
#include "flat_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Flat Streamer Entity
|
||||
*/
|
||||
class FlatStreamerEntity {
|
||||
public:
|
||||
typedef std::shared_ptr<FlatStreamerEntity> Pointer;
|
||||
|
||||
//! Constructor
|
||||
explicit FlatStreamerEntity(IndexStreamer::Stats &stats);
|
||||
|
||||
//! Destructor
|
||||
virtual ~FlatStreamerEntity(void) = default;
|
||||
|
||||
//! Open the entity with storage
|
||||
int open(IndexStorage::Pointer storage, const IndexMeta &mt);
|
||||
|
||||
//! Close the entity
|
||||
int close(void);
|
||||
|
||||
//! Flush Linear Meta information to storage
|
||||
int flush_linear_meta(void);
|
||||
|
||||
//! Flush linear index to storage
|
||||
int flush(uint64_t checkpoint);
|
||||
|
||||
//! Add vector to linear index
|
||||
int add(uint64_t key, const void *vec, size_t size);
|
||||
|
||||
//! Search in linear list with filter
|
||||
int search(const void *query, const IndexFilter &filter, uint32_t *scan_count,
|
||||
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const;
|
||||
|
||||
//! Search in a block
|
||||
void search_block(const void *query, const BlockLocation &bl,
|
||||
const BlockHeader *hd, float norm_val,
|
||||
IndexDocumentHeap *heap) const;
|
||||
|
||||
//! Search in a block with filter
|
||||
void search_block(const void *query, const BlockLocation &bl,
|
||||
const BlockHeader *hd, float norm_val,
|
||||
const IndexFilter &filter, const DeletionMap *deletion_map,
|
||||
IndexDocumentHeap *heap,
|
||||
IndexContext::Stats *context_stats) const;
|
||||
|
||||
//! Flat Search with filter
|
||||
int search_bf(const void *query, const IndexFilter &filter,
|
||||
IndexDocumentHeap *heap,
|
||||
IndexContext::Stats *context_stats) const;
|
||||
|
||||
//! Clone the entity
|
||||
virtual FlatStreamerEntity::Pointer clone(void) const;
|
||||
|
||||
//! Retrieve the total vectors in the index
|
||||
size_t vector_count(void) const {
|
||||
return meta_.header.total_vector_count;
|
||||
}
|
||||
|
||||
//! Retrieve the linear list count
|
||||
size_t linear_list_count(void) const {
|
||||
return meta_.header.linear_list_count;
|
||||
}
|
||||
|
||||
//! Retrieve block size of the linear vector
|
||||
size_t linear_block_size(void) const {
|
||||
return meta_.header.block_size;
|
||||
}
|
||||
|
||||
//! Retrieve the vectors count in one block
|
||||
size_t block_vector_count(void) const {
|
||||
// assert(meta_.header.block_vector_count == 32);
|
||||
return meta_.header.block_vector_count;
|
||||
}
|
||||
|
||||
//! Retrieve IndexMeta of the linear index
|
||||
const IndexMeta &meta(void) const {
|
||||
return index_meta_;
|
||||
}
|
||||
|
||||
//! Retrieve mutable IndexMeta of the linear index
|
||||
IndexMeta *mutable_meta(void) {
|
||||
return &index_meta_;
|
||||
}
|
||||
|
||||
//! Retrieve vector by local id
|
||||
const void *get_vector_by_key(uint64_t key) const;
|
||||
|
||||
int get_vector_by_key(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const;
|
||||
|
||||
//! Create a new iterator
|
||||
IndexProvider::Iterator::Pointer creater_iterator(void) const;
|
||||
|
||||
|
||||
//! Set params
|
||||
void set_block_vector_count(uint32_t count) {
|
||||
meta_.header.block_vector_count = count;
|
||||
}
|
||||
|
||||
void set_use_key_info_map(bool use_id_map) {
|
||||
use_key_info_map_ = use_id_map;
|
||||
LOG_DEBUG("use_key_info_map_: %d", (int)use_key_info_map_);
|
||||
}
|
||||
|
||||
//! Set params
|
||||
void set_segment_size(uint32_t size) {
|
||||
meta_.segment_size = size;
|
||||
}
|
||||
|
||||
//! Set params
|
||||
void set_linear_list_count(uint32_t count) {
|
||||
meta_.header.linear_list_count = count;
|
||||
}
|
||||
|
||||
//! Set params
|
||||
void enable_filter_same_key(bool enabled) {
|
||||
filter_same_key_ = enabled;
|
||||
}
|
||||
|
||||
inline uint64_t key(uint32_t id) const {
|
||||
if (id < id_key_vector_.size()) {
|
||||
return id_key_vector_[id];
|
||||
} else {
|
||||
return kInvalidKey;
|
||||
}
|
||||
}
|
||||
|
||||
inline void row_major_distance(const void *query, const void *feature,
|
||||
size_t fnum, float *out) const {
|
||||
const uint8_t *cur_feature = reinterpret_cast<const uint8_t *>(feature);
|
||||
for (size_t f = 0; f < fnum; ++f) {
|
||||
row_distance_(query, cur_feature, index_meta_.dimension(), out + f);
|
||||
cur_feature += index_meta_.element_size();
|
||||
}
|
||||
}
|
||||
|
||||
int add_vector_with_id(const uint32_t id, const void *query,
|
||||
const uint32_t element_size);
|
||||
|
||||
private:
|
||||
//! Disable them
|
||||
FlatStreamerEntity(const FlatStreamerEntity &) = delete;
|
||||
FlatStreamerEntity &operator=(const FlatStreamerEntity &) = delete;
|
||||
|
||||
/*! Iterator of all the linear list
|
||||
*/
|
||||
class Iterator : public IndexProvider::Iterator {
|
||||
public:
|
||||
//! Constructor
|
||||
Iterator(const FlatStreamerEntity::Pointer &entity) : entity_(entity) {
|
||||
this->read_next_block();
|
||||
}
|
||||
//! Retrieve pointer of data
|
||||
const void *data(void) const override {
|
||||
return reinterpret_cast<const char *>(data_) +
|
||||
block_vector_index_ * entity_->index_meta_.element_size();
|
||||
}
|
||||
//! Test if the iterator is valid
|
||||
bool is_valid(void) const override {
|
||||
return is_valid_;
|
||||
}
|
||||
//! Retrieve primary key
|
||||
uint64_t key(void) const override {
|
||||
return keys_[block_vector_index_];
|
||||
}
|
||||
//! Next iterator
|
||||
void next(void) override {
|
||||
if (++block_vector_index_ == block_vector_count_) {
|
||||
++block_index_;
|
||||
this->read_next_block();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
//! Read next non-empty block
|
||||
void read_next_block(void);
|
||||
|
||||
//! Members
|
||||
std::string buffer_{};
|
||||
const FlatStreamerEntity::Pointer entity_;
|
||||
IndexStorage::MemoryBlock block_;
|
||||
const void *data_{nullptr};
|
||||
const uint64_t *keys_{nullptr};
|
||||
uint32_t segment_id_{1u}; // The first segment is header info
|
||||
uint32_t block_index_{0u};
|
||||
uint32_t block_vector_index_{0u};
|
||||
uint32_t block_vector_count_{0u};
|
||||
bool is_valid_{true};
|
||||
};
|
||||
|
||||
//! Retrive storage segment by index
|
||||
const IndexStorage::Segment::Pointer get_segment(size_t index) const {
|
||||
for (size_t i = segments_.size(); i <= index; ++i) {
|
||||
auto segment_id =
|
||||
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
|
||||
auto segment = storage_->get(segment_id);
|
||||
if (!segment) {
|
||||
LOG_ERROR("Failed to get segment %s", segment_id.c_str());
|
||||
return IndexStorage::Segment::Pointer();
|
||||
}
|
||||
segments_.emplace_back(std::move(segment));
|
||||
}
|
||||
return segments_[index];
|
||||
}
|
||||
|
||||
//! Rejust the segment size as to aligned by page size
|
||||
void AdjustSegmentSize(StreamerLinearMeta *mt) {
|
||||
if (mt->segment_size < mt->header.block_size) {
|
||||
mt->segment_size = mt->header.block_size;
|
||||
}
|
||||
mt->segment_size = ailego_align(
|
||||
mt->segment_size / mt->header.block_size * mt->header.block_size,
|
||||
ailego::MemoryHelper::PageSize());
|
||||
}
|
||||
|
||||
//! Init with an empty storage
|
||||
int init_storage(IndexStorage::Pointer storage);
|
||||
|
||||
//! Load linear meta information from storage
|
||||
int load_linear_meta(IndexStorage::Pointer storage);
|
||||
|
||||
//! Load keys to keys map
|
||||
int load_segment_keys_to_map(BlockLocation block);
|
||||
|
||||
//! Load keys to keys map
|
||||
int load_segment_keys_to_vector(void);
|
||||
|
||||
//! Load index from storage
|
||||
int load_storage(IndexStorage::Pointer storage);
|
||||
|
||||
//! Check whether the block is empty
|
||||
bool is_valid_block(const BlockLocation &block) const {
|
||||
return block.segment_id != 0;
|
||||
}
|
||||
|
||||
//! Update header block of an linear list
|
||||
int update_head_block(const BlockLocation &block) {
|
||||
ailego_assert_with(segments_.size() != 0, "Invalid Segments");
|
||||
|
||||
auto &hd_segment = segments_[0];
|
||||
if (hd_segment->write(0, &block, sizeof(block)) != sizeof(block)) {
|
||||
LOG_ERROR("Failed to write head block location");
|
||||
return IndexError_WriteData;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Alloc a new segment
|
||||
int alloc_segment(void);
|
||||
|
||||
//! Alloc a new block
|
||||
int alloc_block(const BlockLocation &next, BlockLocation *block);
|
||||
|
||||
//! Add a record to a block
|
||||
int add_to_block(const BlockLocation &block, uint64_t key, const void *data,
|
||||
size_t size);
|
||||
|
||||
private:
|
||||
size_t get_block_offset(uint32_t block_index) const {
|
||||
return block_index * linear_block_size();
|
||||
}
|
||||
|
||||
size_t get_block_header_offset(uint32_t block_index) const {
|
||||
return get_block_offset(block_index) + linear_block_size() -
|
||||
sizeof(BlockHeader);
|
||||
}
|
||||
|
||||
size_t get_block_deletion_map_offset(uint32_t block_index) const {
|
||||
return get_block_header_offset(block_index) - sizeof(DeletionMap);
|
||||
}
|
||||
|
||||
size_t get_block_key_offset(uint32_t block_index,
|
||||
uint32_t vector_index) const {
|
||||
return get_block_offset(block_index) +
|
||||
block_vector_count() * index_meta_.element_size() +
|
||||
sizeof(uint64_t) * vector_index;
|
||||
}
|
||||
|
||||
size_t get_block_vector_offset(uint32_t block_index,
|
||||
uint32_t vector_index) const {
|
||||
return this->get_block_offset(block_index) +
|
||||
vector_index * index_meta_.element_size();
|
||||
}
|
||||
|
||||
//! Get header block of an linear list
|
||||
int get_head_block(IndexStorage::MemoryBlock &header_block) const {
|
||||
ailego_assert_with(segments_.size() != 0, "Invalid Segments");
|
||||
auto &hd_segment = segments_[0];
|
||||
if (hd_segment->read(0, header_block, sizeof(BlockLocation)) !=
|
||||
sizeof(BlockLocation)) {
|
||||
LOG_ERROR("Failed to read head block location");
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Get BlockHeader of the block
|
||||
int get_block_header(const BlockLocation &block,
|
||||
IndexStorage::MemoryBlock &header_block) const {
|
||||
// The header is located in the end of a block to align features
|
||||
auto &segment = this->get_segment(block.segment_id);
|
||||
ailego_assert_with(segment != nullptr, "Index Overflow");
|
||||
size_t off = this->get_block_header_offset(block.block_index);
|
||||
if (segment->read(off, header_block, sizeof(BlockHeader)) !=
|
||||
sizeof(BlockHeader)) {
|
||||
LOG_ERROR("Failed to read block header, off=%zu", off);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
int get_block_deletion_map(
|
||||
const BlockLocation &block,
|
||||
IndexStorage::MemoryBlock &deletion_map_block) const {
|
||||
auto &segment = this->get_segment(block.segment_id);
|
||||
ailego_assert_with(segment != nullptr, "Index Overflow");
|
||||
size_t off = this->get_block_deletion_map_offset(block.block_index);
|
||||
if (segment->read(off, deletion_map_block, sizeof(DeletionMap)) !=
|
||||
sizeof(DeletionMap)) {
|
||||
LOG_ERROR("Failed to read deletion map, off=%zu", off);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int get_block_keys(const BlockLocation &block,
|
||||
IndexStorage::MemoryBlock &keys_block) const {
|
||||
auto &segment = this->get_segment(block.segment_id);
|
||||
ailego_assert_with(segment != nullptr, "Index Overflow");
|
||||
size_t off = this->get_block_key_offset(block.block_index, 0);
|
||||
if (segment->read(off, keys_block,
|
||||
block_vector_count() * sizeof(uint64_t)) !=
|
||||
block_vector_count() * sizeof(uint64_t)) {
|
||||
LOG_ERROR("Failed to read block header, off=%zu", off);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int get_block_vectors(const BlockLocation &block,
|
||||
IndexStorage::MemoryBlock &vector_block) const {
|
||||
auto &segment = this->get_segment(block.segment_id);
|
||||
ailego_assert_with(segment != nullptr, "Index Overflow");
|
||||
size_t off = this->get_block_vector_offset(block.block_index, 0);
|
||||
if (segment->read(off, vector_block,
|
||||
block_vector_count() * index_meta_.element_size()) !=
|
||||
block_vector_count() * index_meta_.element_size()) {
|
||||
LOG_ERROR("Failed to read block header, off=%zu", off);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Constants
|
||||
static constexpr size_t kMaxSegmentId = std::numeric_limits<uint16_t>::max();
|
||||
static constexpr size_t kMaxBlockId = std::numeric_limits<uint16_t>::max();
|
||||
|
||||
//! Members
|
||||
std::mutex mutex_{};
|
||||
IndexMeta index_meta_{};
|
||||
IndexStorage::Pointer storage_{};
|
||||
IndexMetric::MatrixDistance row_distance_{}, column_distance_{};
|
||||
mutable std::vector<IndexStorage::Segment::Pointer> segments_{};
|
||||
StreamerLinearMeta meta_{};
|
||||
IndexStreamer::Stats &stats_;
|
||||
mutable std::shared_ptr<ailego::SharedMutex> key_info_map_lock_{};
|
||||
std::unordered_map<uint64_t, VectorLocation> key_info_map_{};
|
||||
std::vector<VectorLocation> withid_key_info_map_{};
|
||||
std::vector<uint32_t> withid_key_map_{};
|
||||
std::vector<uint64_t> id_key_vector_{};
|
||||
bool filter_same_key_{false};
|
||||
bool use_key_info_map_{true};
|
||||
uint32_t vec_unit_size_{0};
|
||||
uint32_t vec_cols_{0};
|
||||
mutable std::string vec_buf_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "flat_distance_matrix.h"
|
||||
#include "flat_searcher.h"
|
||||
#include "flat_streamer.h"
|
||||
#include "flat_utility.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Brute Force Streamer Provider
|
||||
*/
|
||||
|
||||
template <size_t BATCH_SIZE>
|
||||
class FlatStreamerProvider : public IndexProvider {
|
||||
public:
|
||||
//! Constructor
|
||||
FlatStreamerProvider(const FlatStreamer<BATCH_SIZE> *owner) {
|
||||
feature_size_ = owner->meta().element_size();
|
||||
total_vector_count_ = owner->entity().vector_count();
|
||||
owner_ = owner;
|
||||
block_buffer_.resize(BATCH_SIZE * feature_size_);
|
||||
}
|
||||
|
||||
//! Create a new iterator
|
||||
IndexProvider::Iterator::Pointer create_iterator(void) override {
|
||||
return owner_->entity().creater_iterator();
|
||||
}
|
||||
|
||||
//! Retrieve count of vectors
|
||||
size_t count(void) const override {
|
||||
return total_vector_count_;
|
||||
}
|
||||
|
||||
//! Retrieve dimension of vector
|
||||
size_t dimension(void) const override {
|
||||
return owner_->meta().dimension();
|
||||
}
|
||||
|
||||
//! Retrieve type of vector
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return owner_->meta().data_type();
|
||||
}
|
||||
|
||||
//! Retrieve vector size in bytes
|
||||
size_t element_size(void) const override {
|
||||
return owner_->meta().element_size();
|
||||
}
|
||||
|
||||
//! Retrieve a vector using a primary key
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
return this->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return this->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
//! Retrieve the owner class
|
||||
const std::string &owner_class(void) const override {
|
||||
return owner_->name();
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Retrieve a vector via primary key
|
||||
const void *get_vector_by_key(uint64_t key) const {
|
||||
return owner_->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
int get_vector_by_key(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const {
|
||||
return owner_->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
const FlatStreamer<BATCH_SIZE> *owner_{nullptr};
|
||||
IndexStorage::Segment::Pointer features_segment_{};
|
||||
uint32_t feature_size_{0};
|
||||
uint32_t total_vector_count_{0};
|
||||
mutable std::vector<uint8_t> block_buffer_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,201 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <ailego/utility/matrix_helper.h>
|
||||
#include <ailego/utility/time_helper.h>
|
||||
#include "framework/index_error.h"
|
||||
#include "framework/index_factory.h"
|
||||
#include "framework/index_meta.h"
|
||||
#include "framework/index_metric.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! The default size of reading a block
|
||||
static constexpr uint32_t FLAT_READ_BLOCK_SIZE = 4 * 1024 * 1024;
|
||||
static const std::string FLAT_LINEAR_META_SEG_ID = "linear.linear_meta";
|
||||
static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID =
|
||||
"linear.linear_list_head";
|
||||
|
||||
// reducer params
|
||||
static const std::string PARAM_FLAT_REDUCER_WORKING_PATH(
|
||||
"proxima.linear.reducer.working_path");
|
||||
static const std::string PARAM_FLAT_REDUCER_NUM_OF_ADD_THREADS(
|
||||
"proxima.linear.reducer.num_of_add_threads");
|
||||
static const std::string PARAM_FLAT_REDUCER_INDEX_NAME(
|
||||
"proxima.linear.reducer.index_name");
|
||||
|
||||
// index params
|
||||
static const std::string PARAM_FLAT_SEGMENT_KEYS("linear.keys");
|
||||
static const std::string PARAM_FLAT_SEGMENT_FEATURES("linear.features");
|
||||
static const std::string PARAM_FLAT_SEGMENT_MAPPING("linear.mapping");
|
||||
|
||||
static const std::string FLAT_BUILDER_COLUMN_MAJOR_ORDER(
|
||||
"proxima.linear.builder.column_major_order");
|
||||
static const std::string FLAT_SEARCHER_BATCH_SIZE(
|
||||
"proxima.linear.searcher.batch_size");
|
||||
static const std::string FLAT_SEARCHER_READ_BLOCK_SIZE(
|
||||
"proxima.linear.searcher.read_block_size");
|
||||
static const std::string FLAT_SEARCHER_USE_ID_MAP(
|
||||
"proxima.linear.searcher.use_id_map");
|
||||
|
||||
//! Determines if a number is equal to two to the power of n.
|
||||
template <size_t K>
|
||||
struct IsEqualPowerofTwo
|
||||
: std::integral_constant<bool, K != 0 && (K ^ (K - 1)) == (K | (K - 1))> {};
|
||||
|
||||
//! Transpose a block
|
||||
template <size_t M>
|
||||
static inline void ReverseTranspose(size_t align_size, const void *src,
|
||||
size_t dim, void *dst) {
|
||||
switch (align_size) {
|
||||
case 2:
|
||||
ailego::MatrixHelper::ReverseTranspose<uint16_t, M>(src, dim, dst);
|
||||
break;
|
||||
case 4:
|
||||
ailego::MatrixHelper::ReverseTranspose<uint32_t, M>(src, dim, dst);
|
||||
break;
|
||||
case 8:
|
||||
ailego::MatrixHelper::ReverseTranspose<uint64_t, M>(src, dim, dst);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void ReverseTranspose(size_t align_size, const void *src,
|
||||
size_t m, size_t dim, void *dst) {
|
||||
switch (align_size) {
|
||||
case 2:
|
||||
ailego::MatrixHelper::ReverseTranspose<uint16_t>(src, m, dim, dst);
|
||||
break;
|
||||
case 4:
|
||||
ailego::MatrixHelper::ReverseTranspose<uint32_t>(src, m, dim, dst);
|
||||
break;
|
||||
case 8:
|
||||
ailego::MatrixHelper::ReverseTranspose<uint64_t>(src, m, dim, dst);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline void TransposeOne(const void *src, size_t M, size_t N,
|
||||
void *dst) {
|
||||
for (size_t i = 0; i < N; ++i) {
|
||||
reinterpret_cast<T *>(dst)[i] = reinterpret_cast<const T *>(src)[i * M];
|
||||
}
|
||||
}
|
||||
|
||||
static inline void Transpose(size_t align_size, const void *src, size_t m,
|
||||
size_t dim, void *dst) {
|
||||
switch (align_size) {
|
||||
case 2:
|
||||
ailego::MatrixHelper::Transpose<uint16_t>(src, m, dim, dst);
|
||||
break;
|
||||
case 4:
|
||||
ailego::MatrixHelper::Transpose<uint32_t>(src, m, dim, dst);
|
||||
break;
|
||||
case 8:
|
||||
ailego::MatrixHelper::Transpose<uint64_t>(src, m, dim, dst);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
//! Transpose queries
|
||||
template <size_t K>
|
||||
void TransposeQueries(const void *query, const IndexQueryMeta &qmeta,
|
||||
size_t query_count, std::string *out) {
|
||||
if (K <= 1) {
|
||||
ailego_assert(query_count == 1);
|
||||
(void)query_count;
|
||||
out->append(reinterpret_cast<const char *>(query) + out->size(),
|
||||
qmeta.element_size());
|
||||
} else {
|
||||
ailego_assert_with(IsEqualPowerofTwo<K>::value,
|
||||
"K must be equal to two to the power of n.");
|
||||
|
||||
size_t query_batch_count = query_count / K;
|
||||
size_t query_offset = out->size();
|
||||
out->resize(query_offset + query_batch_count * K * qmeta.element_size());
|
||||
|
||||
switch (IndexMeta::AlignSizeof(qmeta.data_type())) {
|
||||
case 2:
|
||||
for (size_t i = 0; i != query_batch_count; ++i) {
|
||||
ailego::MatrixHelper::Transpose<uint16_t, K>(
|
||||
(const char *)query + query_offset,
|
||||
qmeta.element_size() / sizeof(uint16_t), &((*out)[query_offset]));
|
||||
query_offset += qmeta.element_size() * K;
|
||||
}
|
||||
break;
|
||||
|
||||
case 4:
|
||||
for (size_t i = 0; i != query_batch_count; ++i) {
|
||||
ailego::MatrixHelper::Transpose<uint32_t, K>(
|
||||
(const char *)query + query_offset,
|
||||
qmeta.element_size() / sizeof(uint32_t), &((*out)[query_offset]));
|
||||
|
||||
query_offset += qmeta.element_size() * K;
|
||||
}
|
||||
break;
|
||||
|
||||
case 8:
|
||||
for (size_t i = 0; i != query_batch_count; ++i) {
|
||||
ailego::MatrixHelper::Transpose<uint64_t, K>(
|
||||
(const char *)query + query_offset,
|
||||
qmeta.element_size() / sizeof(uint64_t), &((*out)[query_offset]));
|
||||
query_offset += qmeta.element_size() * K;
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
ailego_check_with(0, "BAD CASE");
|
||||
}
|
||||
size_t query_left_count = query_count % K;
|
||||
if (query_left_count != 0) {
|
||||
TransposeQueries<(K >> 1)>(query, qmeta, query_left_count, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//! Create and initialize measure
|
||||
static inline int InitializeMetric(const IndexMeta &mt,
|
||||
IndexMetric::Pointer *out) {
|
||||
IndexMetric::Pointer measure = IndexFactory::CreateMetric(mt.metric_name());
|
||||
if (!measure) {
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
int error_code = measure->init(mt, mt.metric_params());
|
||||
if (error_code != 0) {
|
||||
return error_code;
|
||||
}
|
||||
*out = measure;
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Verify measure
|
||||
static inline bool VerifyMetric(const IndexMeta &meta) {
|
||||
IndexMetric::Pointer measure = IndexFactory::CreateMetric(meta.metric_name());
|
||||
if (!measure) {
|
||||
return false;
|
||||
}
|
||||
int error_code = measure->init(meta, meta.metric_params());
|
||||
if (error_code != 0) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
Reference in New Issue
Block a user