Initial commit

This commit is contained in:
sanyi
2025-12-30 11:02:17 +08:00
commit 6524fabd80
841 changed files with 293919 additions and 0 deletions
+11
View File
@@ -0,0 +1,11 @@
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
#message(STATUS "PROJECT_ROOT_DIR = ${PROJECT_ROOT_DIR}")
cc_library(
NAME core_knn_flat
STATIC SHARED STRICT ALWAYS_LINK
SRCS *.cc
LIBS core_framework
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm ${PROJECT_ROOT_DIR}/src/core/framework
VERSION "${PROXIMA_ZVEC_VERSION}"
)
+284
View File
@@ -0,0 +1,284 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "flat_builder.h"
#include "flat_utility.h"
namespace zvec {
namespace core {
template <size_t BATCH_SIZE>
int FlatBuilder<BATCH_SIZE>::init(const IndexMeta &meta,
const ailego::Params &params) {
meta_ = meta;
// Set the major order
bool column_major_order = false;
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order)) {
meta_.set_major_order(column_major_order ? IndexMeta::MO_COLUMN
: IndexMeta::MO_ROW);
}
// Verify column major order
if (meta_.major_order() != IndexMeta::MO_ROW) {
IndexMeta::DataType dt = meta_.data_type();
bool support_column_major = false;
if ((dt != IndexMeta::DataType::DT_FP32 &&
dt != IndexMeta::DataType::DT_FP16 &&
dt != IndexMeta::DataType::DT_INT8 && dt != IndexMeta::DT_INT4 &&
dt != IndexMeta::DT_BINARY32 && dt != IndexMeta::DT_BINARY64) ||
(meta_.unit_size() != IndexMeta::UnitSizeof(dt))) {
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
LOG_ERROR("Unsupported type %d with unit size %u.", dt,
meta_.unit_size());
return IndexError_Unsupported;
} else {
support_column_major = false;
}
}
if (meta_.element_size() % IndexMeta::AlignSizeof(dt) != 0) {
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
LOG_ERROR("Unsupported type %d with dimension %u.", dt,
meta_.dimension());
return IndexError_Unsupported;
} else {
support_column_major = false;
}
}
if (meta_.major_order() == IndexMeta::MO_UNDEFINED &&
support_column_major) {
meta_.set_major_order(IndexMeta::MO_COLUMN);
}
}
if (!VerifyMetric(meta_)) {
LOG_ERROR("Invalid index measure %s.", meta_.metric_name().c_str());
return IndexError_InvalidArgument;
}
std::string tag = std::to_string(BATCH_SIZE);
ailego::Params searcher_params;
searcher_params.set(FLAT_SEARCHER_BATCH_SIZE, BATCH_SIZE);
meta_.set_searcher("FlatSearcher" + tag, 0, searcher_params);
meta_.set_builder("FlatBuilder" + tag, 0, params);
return 0;
}
template <size_t BATCH_SIZE>
int FlatBuilder<BATCH_SIZE>::build(IndexThreads::Pointer,
IndexHolder::Pointer holder) {
ailego::ElapsedTime stamp;
if (!holder->is_matched(meta_)) {
LOG_ERROR("The holder is unmatched with initialized meta.");
return IndexError_Mismatch;
}
holder_ = std::move(holder);
stats_.set_built_count(holder_->count());
stats_.set_built_costtime(stamp.milli_seconds());
return 0;
}
template <size_t BATCH_SIZE>
int FlatBuilder<BATCH_SIZE>::dump(const IndexDumper::Pointer &dumper) {
ailego::ElapsedTime stamp;
if (!holder_) {
return IndexError_NoReady;
}
std::vector<uint64_t> keys;
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
int error_code = this->write_column_index(dumper.get(), &keys);
if (error_code != 0) {
return error_code;
}
} else {
int error_code = this->write_row_index(dumper.get(), &keys);
if (error_code != 0) {
return error_code;
}
}
int error_code = this->write_keys(keys, dumper.get());
if (error_code != 0) {
return error_code;
}
error_code = this->write_mapping(keys, dumper.get());
if (error_code != 0) {
return error_code;
}
error_code = IndexHelper::SerializeToDumper(meta_, dumper.get());
if (error_code != 0) {
return error_code;
}
holder_ = nullptr;
stats_.set_dumped_count(keys.size());
stats_.set_dumped_costtime(stamp.milli_seconds());
return 0;
}
template <size_t BATCH_SIZE>
int FlatBuilder<BATCH_SIZE>::write_keys(const std::vector<uint64_t> &keys,
IndexDumper *dumper) {
size_t keys_size = keys.size() * sizeof(uint64_t);
size_t keys_padding_size = ailego_align(keys_size, 32) - keys_size;
if (dumper->write(keys.data(), keys_size) != keys_size) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
// Write the padding if need
if (keys_padding_size) {
std::string padding(keys_padding_size, '\0');
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
0);
}
template <size_t BATCH_SIZE>
int FlatBuilder<BATCH_SIZE>::write_mapping(const std::vector<uint64_t> &keys,
IndexDumper *dumper) {
std::vector<uint32_t> mapping(keys.size());
std::iota(mapping.begin(), mapping.end(), 0);
std::sort(
mapping.begin(), mapping.end(),
[&keys](uint32_t lhs, uint32_t rhs) { return (keys[lhs] < keys[rhs]); });
size_t mapping_size = mapping.size() * sizeof(uint32_t);
size_t mapping_padding_size = ailego_align(mapping_size, 32) - mapping_size;
if (dumper->write(mapping.data(), mapping_size) != mapping_size) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
// Write the padding if need
if (mapping_padding_size) {
std::string padding(mapping_padding_size, '\0');
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
mapping_padding_size, 0);
}
template <size_t BATCH_SIZE>
template <typename T>
int FlatBuilder<BATCH_SIZE>::write_column_index(IndexDumper *dumper,
std::vector<uint64_t> *keys) {
auto iter = holder_->create_iterator();
if (!iter) {
LOG_ERROR("Failed to create iterator of holder");
return IndexError_Runtime;
}
// Write features
size_t element_size = holder_->element_size();
size_t block_size = element_size * BATCH_SIZE;
std::string block1, block2;
block1.reserve(block_size);
block2.reserve(block_size);
for (; iter->is_valid(); iter->next()) {
block1.append(reinterpret_cast<const char *>(iter->data()), element_size);
keys->emplace_back(iter->key());
if (block1.size() == block_size) {
ailego::MatrixHelper::Transpose<T, BATCH_SIZE>(
block1.data(), element_size / sizeof(T), (void *)block2.data());
if (dumper->write(block2.data(), block_size) != block_size) {
LOG_ERROR("Failed to write data into dumper %s",
dumper->name().c_str());
return IndexError_WriteData;
}
block1.clear();
}
}
if (!block1.empty()) {
if (dumper->write(block1.data(), block1.size()) != block1.size()) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
}
// Write the padding if need
size_t features_size = keys->size() * element_size;
size_t features_padding_size =
ailego_align(features_size, 32) - features_size;
if (features_padding_size) {
std::string padding(features_padding_size, '\0');
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
features_padding_size, 0);
}
template <size_t BATCH_SIZE>
int FlatBuilder<BATCH_SIZE>::write_row_index(IndexDumper *dumper,
std::vector<uint64_t> *keys) {
auto iter = holder_->create_iterator();
if (!iter) {
LOG_ERROR("Failed to create iterator of holder");
return IndexError_Runtime;
}
// Write features
size_t element_size = holder_->element_size();
for (; iter->is_valid(); iter->next()) {
if (dumper->write(iter->data(), element_size) != element_size) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
keys->emplace_back(iter->key());
}
// Write the padding if need
size_t features_size = keys->size() * element_size;
size_t features_padding_size =
ailego_align(features_size, 32) - features_size;
if (features_padding_size) {
std::string padding(features_padding_size, '\0');
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
features_padding_size, 0);
}
INDEX_FACTORY_REGISTER_BUILDER_ALIAS(LinearBuilder, FlatBuilder<32>);
INDEX_FACTORY_REGISTER_BUILDER_ALIAS(FlatBuilder, FlatBuilder<32>);
INDEX_FACTORY_REGISTER_BUILDER_ALIAS(FlatBuilder16, FlatBuilder<16>);
INDEX_FACTORY_REGISTER_BUILDER_ALIAS(FlatBuilder32, FlatBuilder<32>);
} // namespace core
} // namespace zvec
+101
View File
@@ -0,0 +1,101 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <framework/index_builder.h>
#include <framework/index_helper.h>
#include "flat_utility.h"
namespace zvec {
namespace core {
/*! Flat Builder
*/
template <size_t BATCH_SIZE>
class FlatBuilder : public IndexBuilder {
public:
//! Destructor
virtual ~FlatBuilder(void) {}
//! Initialize the builder
int init(const IndexMeta &meta, const ailego::Params &params) override;
//! Cleanup the builder
int cleanup(void) override {
holder_ = nullptr;
return 0;
}
//! Train the data
int train(IndexThreads::Pointer, IndexHolder::Pointer) override {
stats_.set_trained_count(0u);
stats_.set_trained_costtime(0u);
return 0;
}
//! Train the data
int train(const IndexTrainer::Pointer &) override {
stats_.set_trained_count(0u);
stats_.set_trained_costtime(0u);
return 0;
}
//! Build the index
int build(IndexThreads::Pointer, IndexHolder::Pointer holder) override;
//! Dump index into storage
int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
const IndexBuilder::Stats &stats(void) const override {
return stats_;
}
protected:
//! Dump index keys
int write_keys(const std::vector<uint64_t> &keys, IndexDumper *dumper);
//! Dump index keys mapping
int write_mapping(const std::vector<uint64_t> &keys, IndexDumper *dumper);
//! Dump index using column-major-order format
template <typename T>
int write_column_index(IndexDumper *dumper, std::vector<uint64_t> *keys);
//! Dump index using column-major-order format
int write_column_index(IndexDumper *dumper, std::vector<uint64_t> *keys) {
switch (IndexMeta::AlignSizeof(meta_.data_type())) {
case 2:
return this->write_column_index<uint16_t>(dumper, keys);
case 4:
return this->write_column_index<uint32_t>(dumper, keys);
case 8:
return this->write_column_index<uint64_t>(dumper, keys);
default:
ailego_check_with(0, "BAD CASE");
}
return IndexError_Runtime;
}
//! Dump index using row-major-order format
int write_row_index(IndexDumper *dumper, std::vector<uint64_t> *keys);
private:
IndexMeta meta_{};
IndexBuilder::Stats stats_{};
IndexHolder::Pointer holder_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,184 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "flat_utility.h"
namespace zvec {
namespace core {
/*! Brute Force Distance Tuple
*/
template <size_t K, typename = void>
class FlatDistanceTuple;
/*! Brute Force Distance Tuple
*/
template <>
class FlatDistanceTuple<1> {
public:
//! Retrieve non-zero if all distances are valid.
bool is_valid(void) const {
return !!distance_;
}
//! Retrieve non-zero if a distance is valid.
bool is_valid(size_t m) const {
return m == 1 && !!distance_;
}
//! Initialize the distance tuple
void initialize(const IndexMetric &measure) {
distance_ = measure.distance_matrix(1, 1);
}
//! Initialize the distance tuple
void initialize(const IndexMetric &measure, size_t m) {
distance_ = measure.distance_matrix(m, 1);
}
//! Compute the distance between matrix and query
template <size_t M>
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
typename std::enable_if<M == 1>::type {
distance_(m, q, dim, out);
}
private:
IndexMetric::MatrixDistance distance_{};
};
/*! Brute Force Distance Tuple
*/
template <size_t K>
class FlatDistanceTuple<
K, typename std::enable_if<IsEqualPowerofTwo<K>::value>::type> {
public:
//! Retrieve non-zero if all distances are valid.
bool is_valid(void) const {
return (distance_tuple_.is_valid() && !!distance_);
}
//! Retrieve non-zero if a distance is valid.
bool is_valid(size_t m) const {
return (m == K ? (!!distance_)
: (m < K ? distance_tuple_.is_valid(m) : false));
}
//! Initialize the distance tuple
void initialize(const IndexMetric &measure) {
distance_tuple_.initialize(measure);
distance_ = measure.distance_matrix(K, 1);
}
//! Initialize the distance tuple
void initialize(const IndexMetric &measure, size_t m) {
distance_tuple_.initialize(measure, m);
distance_ = measure.distance_matrix(m, K);
}
//! Compute the distance between matrix and query
template <size_t M>
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
typename std::enable_if<K == M>::type {
distance_(m, q, dim, out);
}
//! Compute the distance between matrix and query
template <size_t M>
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
typename std::enable_if<(K > M) && IsEqualPowerofTwo<M>::value>::type {
distance_tuple_.template distance<M>(m, q, dim, out);
}
private:
FlatDistanceTuple<(K >> 1)> distance_tuple_{};
IndexMetric::MatrixDistance distance_{};
};
/*! Brute Force Distance Matrix
*/
template <size_t K, typename = void>
class FlatDistanceMatrix;
/*! Brute Force Distance Matrix
*/
template <>
class FlatDistanceMatrix<1> {
public:
//! Retrieve non-zero if all distances are valid.
bool is_valid(void) const {
return (!!distance_);
}
//! Initialize the distance tuple
void initialize(const IndexMetric &measure) {
distance_ = measure.distance_matrix(1, 1);
}
//! Compute the distance between matrix and query
template <size_t M, size_t N = 1u>
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
typename std::enable_if<M == 1u && N == 1u>::type {
distance_(m, q, dim, out);
}
private:
IndexMetric::MatrixDistance distance_{};
};
/*! Brute Force Distance Matrix
*/
template <size_t K>
class FlatDistanceMatrix<
K, typename std::enable_if<IsEqualPowerofTwo<K>::value>::type> {
public:
//! Retrieve non-zero if all distances are valid.
bool is_valid(void) const {
return (tuple_h_.is_valid() && tuple_v_.is_valid());
}
//! Retrieve non-zero if a distance is valid.
bool is_valid(size_t m, size_t n) const {
return (m == K ? tuple_h_.is_valid(n)
: (m < K && n == 1 ? tuple_v_.is_valid(m) : false));
}
//! Initialize the distance tuple
void initialize(const IndexMetric &measure) {
tuple_h_.initialize(measure, K);
tuple_v_.initialize(measure);
}
//! Compute the distance between matrix and query
template <size_t M, size_t N>
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
typename std::enable_if<(K == M) && (K >= N)>::type {
tuple_h_.template distance<N>(m, q, dim, out);
}
//! Compute the distance between matrix and query
template <size_t M, size_t N = 1u>
auto distance(const void *m, const void *q, size_t dim, float *out) const ->
typename std::enable_if<(K > M) && (N == 1u)>::type {
tuple_v_.template distance<M>(m, q, dim, out);
}
private:
FlatDistanceTuple<K> tuple_h_{};
FlatDistanceTuple<(K >> 1)> tuple_v_{};
};
} // namespace core
} // namespace zvec
+184
View File
@@ -0,0 +1,184 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/container/bitmap.h>
namespace zvec {
namespace core {
using node_id_t = uint32_t;
using key_t = uint64_t;
using level_t = int32_t;
using dist_t = float;
using TopkHeap = ailego::KeyValueHeap<node_id_t, dist_t>;
using CandidateHeap =
ailego::KeyValueHeap<node_id_t, dist_t, std::greater<dist_t>>;
constexpr node_id_t kInvalidNodeId = static_cast<node_id_t>(-1);
constexpr key_t kInvalidKey = static_cast<key_t>(-1);
/*! Index Format of Linear Index Header
*/
struct LinearIndexHeader {
LinearIndexHeader()
: header_size(0),
total_vector_count(0),
linear_body_size(0),
linear_list_count(0),
block_vector_count(0),
block_size(0),
block_count(0),
index_meta_size(0) {
memset(reserved_, 0, sizeof(reserved_));
memset(index_meta, 0, sizeof(index_meta));
}
uint32_t header_size{0};
uint32_t total_vector_count{0};
uint64_t linear_body_size{0};
uint32_t linear_list_count{0};
uint32_t block_vector_count{0};
uint32_t block_size{0};
uint32_t block_count{0};
uint32_t index_meta_size{0};
char reserved_[28] = {0};
char index_meta[0];
};
/*! Index Format of Linear Index Meta for each Linear list
*/
struct LinearListMeta {
LinearListMeta() : offset(0), block_count(0), vector_count(0), id_offset(0) {
memset(reserved_, 0, sizeof(reserved_));
}
uint64_t offset{0};
uint32_t block_count{0};
uint32_t vector_count{0};
uint32_t id_offset{0};
char reserved_[16] = {0};
};
/*! Index Format of Location in Linear Index for each vector
*/
struct LinearVecLocation {
LinearVecLocation(size_t off, bool col)
: offset(off), column_major(col), reserved(0u) {}
uint64_t offset : 48; // feature offset in posting block segment
uint64_t column_major : 1; // coloum major if true
uint64_t reserved : 15;
};
/*! Index Format of Integer Quantizer params for each linear list
*/
struct LinearIntegerQuantizerParams {
float scale{1.0};
float bias{0.0};
};
/*! Location of Vectors Block in Storage Segment
*/
struct BlockLocation {
uint16_t segment_id{0};
uint16_t block_index{0};
};
/*! The Header of a Block in Storage Segment
*/
struct BlockHeader {
BlockHeader() : vector_count(0u), column_major(0u), reserved(0u) {}
BlockLocation next;
uint16_t vector_count{0};
uint16_t column_major : 1;
uint16_t reserved : 15;
};
struct DeletionMap {
void set(uint32_t index) {
bitset.set(index);
}
void reset(uint32_t index) {
bitset.reset(index);
}
bool test(uint32_t index) const {
return bitset.test(index);
}
bool is_dirty() const {
return bitset.test_any();
}
ailego::FixedBitset<32> bitset{};
};
static_assert(sizeof(DeletionMap) == 4, "DeletionMap must be 4 bytes");
/*! Meta Information of Streamer Entity
*/
struct StreamerLinearMeta {
StreamerLinearMeta()
: create_time(0),
update_time(0),
revision_id(0),
segment_count(0),
segment_size(0) {
memset(reserved_, 0, sizeof(reserved_));
}
uint64_t create_time{0};
uint64_t update_time{0};
uint64_t revision_id{0};
uint32_t segment_count{0};
uint32_t segment_size{0};
uint8_t reserved_[32] = {0};
LinearIndexHeader header;
};
/*! Location of Vector in Storage Segment
*/
struct VectorLocation {
//! Constructor
VectorLocation(void)
: segment_id(0u), column_major(0u), reserved(0u), offset(0u) {}
//! Constructor
VectorLocation(uint16_t id, bool col, uint32_t off)
: segment_id(id), column_major(col), reserved(0u), offset(off) {}
uint16_t segment_id{0};
uint16_t column_major : 1;
uint16_t reserved : 15;
uint32_t offset{0};
public:
bool operator==(const VectorLocation &other) const {
return segment_id == other.segment_id &&
column_major == other.column_major && offset == other.offset;
}
};
static_assert(sizeof(VectorLocation) == sizeof(uint64_t),
"VectorLocation must be size of 8 bytes");
struct KeyInfo {
KeyInfo(void) : centroid_idx(0u) {}
KeyInfo(uint32_t idx, const VectorLocation &loc)
: centroid_idx(idx), location(loc) {}
KeyInfo(VectorLocation loc) : location(loc) {}
uint32_t centroid_idx{0};
VectorLocation location;
};
} // namespace core
} // namespace zvec
+252
View File
@@ -0,0 +1,252 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "flat_searcher.h"
#include <framework/index_helper.h>
#include <framework/index_searcher.h>
#include "flat_distance_matrix.h"
#include "flat_searcher_context.h"
#include "flat_searcher_provider.h"
#include "flat_utility.h"
namespace zvec {
namespace core {
template <size_t BATCH_SIZE>
IndexProvider::Pointer FlatSearcher<BATCH_SIZE>::create_provider(void) const {
std::lock_guard<std::mutex> lock(mapping_mutex_);
if (mapping_.empty()) {
auto mapping_segment = container_->get(PARAM_FLAT_SEGMENT_MAPPING);
if (!mapping_segment) {
LOG_ERROR("Failed to fetch segment %s",
PARAM_FLAT_SEGMENT_MAPPING.c_str());
return nullptr;
}
if (mapping_segment->data_size() % sizeof(uint32_t) != 0) {
LOG_ERROR("Invalid data size %zu of mapping segment",
mapping_segment->data_size());
return nullptr;
}
size_t mapping_count = mapping_segment->data_size() / sizeof(uint32_t);
if (mapping_count * meta_.element_size() !=
features_segment_->data_size()) {
LOG_ERROR("Invalid data size %zd of mapping segment",
features_segment_->data_size());
return nullptr;
}
const uint32_t *mapping = nullptr;
if (mapping_segment->read(0, reinterpret_cast<const void **>(&mapping),
mapping_segment->data_size()) !=
mapping_segment->data_size()) {
LOG_ERROR("Failed to read data (%zu bytes) from mapping segment",
mapping_segment->data_size());
return nullptr;
}
mapping_.clear();
mapping_.reserve(mapping_count);
std::copy(mapping, mapping + mapping_count, std::back_inserter(mapping_));
}
return IndexProvider::Pointer(new (std::nothrow)
FlatSearcherProvider<BATCH_SIZE>(this));
}
template <size_t BATCH_SIZE>
int FlatSearcher<BATCH_SIZE>::load(IndexStorage::Pointer cntr,
IndexMetric::Pointer measure) {
ailego::ElapsedTime stamp;
if (!cntr) {
return IndexError_InvalidArgument;
}
int error_code = IndexHelper::DeserializeFromStorage(cntr.get(), &meta_);
if (error_code != 0) {
LOG_ERROR(
"Failed to deserialize index meta from container %s, error=%d, %s",
cntr->name().c_str(), error_code, IndexError::What(error_code));
return error_code;
}
if (!measure) {
error_code = InitializeMetric(meta_, &measure_);
if (error_code != 0) {
LOG_ERROR("Failed to initialize index measure %s, error=%d, %s",
meta_.metric_name().c_str(), error_code,
IndexError::What(error_code));
return error_code;
}
if (measure_->query_metric()) {
measure_ = measure_->query_metric();
}
} else {
if (!measure->is_matched(meta_)) {
LOG_ERROR(
"The index measure is unmatched with index meta from container.");
return IndexError_Mismatch;
}
measure_ = std::move(measure);
}
column_major_order_ = (meta_.major_order() == IndexMeta::MO_COLUMN);
distance_matrix_.initialize(*measure_);
if (column_major_order_) {
if (!distance_matrix_.is_valid()) {
LOG_ERROR("Lack of distance functions to support column index.");
return IndexError_Unsupported;
}
} else {
if (!distance_matrix_.is_valid(1, 1)) {
LOG_ERROR("Lack of distance functions to support row index.");
return IndexError_Unsupported;
}
}
auto keys_segment = cntr->get(PARAM_FLAT_SEGMENT_KEYS);
if (!keys_segment) {
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
return IndexError_NoExist;
}
features_segment_ = cntr->get(PARAM_FLAT_SEGMENT_FEATURES);
if (!features_segment_) {
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
return IndexError_NoExist;
}
if (keys_segment->data_size() % sizeof(uint64_t) != 0) {
LOG_ERROR("Invalid data size %zu of keys segment",
keys_segment->data_size());
return IndexError_InvalidLength;
}
size_t keys_count = keys_segment->data_size() / sizeof(uint64_t);
if (keys_count * meta_.element_size() != features_segment_->data_size()) {
LOG_ERROR("Invalid data size %zd of features segment",
features_segment_->data_size());
return IndexError_Mismatch;
}
if (keys_segment->read(0, reinterpret_cast<const void **>(&keys_),
keys_segment->data_size()) !=
keys_segment->data_size()) {
LOG_ERROR("Failed to read data (%zu bytes) from keys segment",
keys_segment->data_size());
return IndexError_ReadData;
}
for (size_t i = 0; i < keys_count; i++) {
key_id_mapping_[keys_[i]] = i;
}
container_ = cntr;
magic_ = IndexContext::GenerateMagic();
stats_.set_loaded_count(keys_count);
stats_.set_loaded_costtime(stamp.milli_seconds());
return 0;
}
template <size_t BATCH_SIZE>
int FlatSearcher<BATCH_SIZE>::search_impl(const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) const {
ailego_assert(query && !!context);
ailego_assert(measure_->is_matched(meta_, qmeta));
FlatSearcherContext<BATCH_SIZE> *bf_context =
dynamic_cast<FlatSearcherContext<BATCH_SIZE> *>(context.get());
if (!bf_context) {
LOG_ERROR("Invalid brute-force searcher context");
return IndexError_InvalidArgument;
}
if (bf_context->magic() != magic_) {
bf_context->reset(this);
}
if (bf_context->group_by_search()) {
return bf_context->group_by_search_impl(query, qmeta, 1);
} else {
return (column_major_order_ ? bf_context->search_column(query, qmeta)
: bf_context->search_row(query, qmeta));
}
}
template <size_t BATCH_SIZE>
int FlatSearcher<BATCH_SIZE>::search_impl(const void *query,
const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const {
ailego_assert(query && count && !!context);
ailego_assert(measure_->is_matched(meta_, qmeta));
FlatSearcherContext<BATCH_SIZE> *bf_context =
dynamic_cast<FlatSearcherContext<BATCH_SIZE> *>(context.get());
if (!bf_context) {
LOG_ERROR("Invalid brute-force searcher context");
return IndexError_InvalidArgument;
}
if (bf_context->magic() != magic_) {
bf_context->reset(this);
}
if (bf_context->group_by_search()) {
return bf_context->group_by_search_impl(query, qmeta, count);
} else {
return (column_major_order_ ? bf_context->search_column(query, qmeta, count)
: bf_context->search_row(query, qmeta, count));
}
}
template <size_t BATCH_SIZE>
int FlatSearcher<BATCH_SIZE>::search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
ailego_assert(query && count && !!context);
ailego_assert(measure_->is_matched(meta_, qmeta));
if (ailego_unlikely(p_keys.size() != count)) {
LOG_ERROR("The size of p_keys is not equal to count");
return IndexError_InvalidArgument;
}
FlatSearcherContext<BATCH_SIZE> *bf_context =
dynamic_cast<FlatSearcherContext<BATCH_SIZE> *>(context.get());
if (!bf_context) {
LOG_ERROR("Invalid brute-force searcher context");
return IndexError_InvalidArgument;
}
if (bf_context->magic() != magic_) {
bf_context->reset(this);
}
return bf_context->search_bf_by_p_keys_impl(query, p_keys, qmeta, count);
}
template <size_t BATCH_SIZE>
IndexSearcher::Context::Pointer FlatSearcher<BATCH_SIZE>::create_context(
void) const {
return IndexSearcher::Context::Pointer(
new FlatSearcherContext<BATCH_SIZE>(this));
}
INDEX_FACTORY_REGISTER_SEARCHER_ALIAS(LinearSearcher, FlatSearcher<32>);
INDEX_FACTORY_REGISTER_SEARCHER_ALIAS(FlatSearcher, FlatSearcher<32>);
INDEX_FACTORY_REGISTER_SEARCHER_ALIAS(FlatSearcher16, FlatSearcher<16>);
INDEX_FACTORY_REGISTER_SEARCHER_ALIAS(FlatSearcher32, FlatSearcher<32>);
} // namespace core
} // namespace zvec
+182
View File
@@ -0,0 +1,182 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/container/params.h>
#include "framework/index_searcher.h"
#include "flat_distance_matrix.h"
#include "flat_index_format.h"
namespace zvec {
namespace core {
/*! Flat Searcher
*/
template <size_t BATCH_SIZE>
class FlatSearcher : public IndexSearcher {
public:
//! Destructor
virtual ~FlatSearcher(void) = default;
//! Initialize Searcher
int init(const ailego::Params &index_params) override {
params_ = index_params;
read_block_size_ = FLAT_READ_BLOCK_SIZE;
index_params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
return 0;
}
//! Cleanup Searcher
int cleanup(void) override {
return this->unload();
}
//! Load index from container
int load(IndexStorage::Pointer cntr, IndexMetric::Pointer measure) override;
//! Unload index
int unload(void) override {
container_ = nullptr;
measure_ = nullptr;
features_segment_ = nullptr;
keys_ = nullptr;
key_id_mapping_.clear();
return 0;
}
//! Similarity brute force search
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override {
return this->search_impl(query, qmeta, context);
}
//! Similarity brute force search
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override {
return this->search_impl(query, qmeta, count, context);
}
//! Similarity search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta,
Context::Pointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const override;
//! Retrieve statistics
const IndexSearcher::Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
const IndexMeta &meta(void) const override {
return meta_;
}
//! Retrieve params of index
const ailego::Params &params(void) const override {
return params_;
}
//! Create a searcher context
IndexSearcher::Context::Pointer create_context(void) const override;
//! Create a searcher provider
IndexProvider::Pointer create_provider(void) const override;
//! Retrieve magic number
uint32_t magic(void) const {
return magic_;
}
//! Retrieve block size of data read
uint32_t read_block_size(void) const {
return read_block_size_;
}
//! Retrieve primary key via index id
uint64_t key(size_t i) const {
return keys_[i];
}
// Retrieve index id via primary key
node_id_t get_id(key_t key) const {
auto it = key_id_mapping_.find(key);
if (it != key_id_mapping_.end()) {
return it->second;
} else {
return kInvalidNodeId;
}
}
//! Retrieve primary key via index id
uint32_t local_index(size_t i) const {
return mapping_[i];
}
//! Retrieve primary key via index id
inline bool column_major_order(void) const {
return column_major_order_;
}
//! Retrieve the distance matrix
const FlatDistanceMatrix<BATCH_SIZE> &distance_matrix(void) const {
return distance_matrix_;
}
//! Clone a features segment
IndexStorage::Segment::Pointer clone_features_segment(void) const {
return features_segment_->clone();
}
const void *get_vector(key_t key) const override {
auto provider = this->create_provider();
return provider->get_vector(key);
}
private:
//! Members
const uint64_t *keys_{nullptr};
std::map<key_t, node_id_t> key_id_mapping_;
uint32_t magic_{IndexContext::GenerateMagic()};
uint32_t read_block_size_{FLAT_READ_BLOCK_SIZE};
bool column_major_order_{false};
IndexMeta meta_{};
IndexStorage::Pointer container_{};
IndexMetric::Pointer measure_{};
ailego::Params params_{};
IndexStorage::Segment::Pointer features_segment_{};
mutable std::vector<uint32_t> mapping_{};
mutable std::mutex mapping_mutex_{};
FlatDistanceMatrix<BATCH_SIZE> distance_matrix_{};
IndexSearcher::Stats stats_{};
};
} // namespace core
} // namespace zvec
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,220 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include "flat_distance_matrix.h"
#include "flat_searcher.h"
// #include "flat_streamer.h"
#include "flat_utility.h"
namespace zvec {
namespace core {
/*! Brute Force Searcher Provider
*/
template <size_t BATCH_SIZE>
class FlatSearcherProvider : public IndexProvider {
public:
//! Constructor
FlatSearcherProvider(const FlatSearcher<BATCH_SIZE> *owner) {
feature_size_ = owner->meta().element_size();
features_segment_ = owner->clone_features_segment();
total_vector_count_ =
features_segment_->data_size() / owner->meta().element_size();
owner_ = owner;
block_buffer_.resize(BATCH_SIZE * feature_size_);
}
//! Create a new iterator
IndexProvider::Iterator::Pointer create_iterator(void) override {
return IndexProvider::Iterator::Pointer(
new (std::nothrow) FlatSearcherProvider::Iterator(owner_));
}
//! Retrieve count of vectors
size_t count(void) const override {
return total_vector_count_;
}
//! Retrieve dimension of vector
size_t dimension(void) const override {
return owner_->meta().dimension();
}
//! Retrieve type of vector
IndexMeta::DataType data_type(void) const override {
return owner_->meta().data_type();
}
//! Retrieve vector size in bytes
size_t element_size(void) const override {
return owner_->meta().element_size();
}
//! Retrieve a vector using a primary key
const void *get_vector(uint64_t key) const override {
return this->get_vector_by_index(owner_->get_id(key));
}
//! Retrieve the owner class
const std::string &owner_class(void) const override {
return owner_->name();
}
protected:
/*! Brute Force Provider Iterator
*/
class Iterator : public IndexProvider::Iterator {
public:
//! Constructor
Iterator(const FlatSearcher<BATCH_SIZE> *owner) {
block_buffer_.resize(BATCH_SIZE * owner->meta().element_size());
feature_size_ = owner->meta().element_size();
features_segment_ = owner->clone_features_segment();
total_vector_count_ =
features_segment_->data_size() / owner->meta().element_size();
owner_ = owner;
cursor_index_ = 0;
offset_ = 0;
this->next_block();
}
//! Retrieve pointer of data
//! NOTICE: the vec feature will be changed after iterating to next, so
//! the caller need to keep a copy of it before iterator to next vector
const void *data(void) const override {
return data_;
}
//! Test if the iterator is valid
bool is_valid(void) const override {
return (!invalid_ && cursor_index_ < total_vector_count_);
}
//! Retrieve primary key
uint64_t key(void) const override {
return owner_->key(cursor_index_);
}
//! Next iterator
void next(void) override {
++cursor_index_;
if ((cursor_index_ % BATCH_SIZE) != 0) {
data_ += feature_size_;
} else {
this->next_block();
}
}
protected:
//! Read a block of data
void next_block(void) {
const void *read_data = nullptr;
size_t read_size = 0;
if (cursor_index_ >= total_vector_count_) {
invalid_ = true;
return;
}
if (cursor_index_ + BATCH_SIZE < total_vector_count_) {
read_size = BATCH_SIZE * feature_size_;
} else {
read_size = (total_vector_count_ - cursor_index_) * feature_size_;
}
if (features_segment_->read(offset_, &read_data, read_size) !=
read_size) {
LOG_ERROR("Failed to read data (%zu bytes) from features segment",
read_size);
invalid_ = true;
return;
}
offset_ += read_size;
// The order of data may be a column format, convert it to the row format.
if (owner_->column_major_order() &&
read_size == BATCH_SIZE * feature_size_) {
uint32_t align_size =
IndexMeta::AlignSizeof(owner_->meta().data_type());
ReverseTranspose<BATCH_SIZE>(align_size, read_data,
feature_size_ / align_size,
&block_buffer_[0]);
data_ = block_buffer_.data();
} else {
data_ = reinterpret_cast<const uint8_t *>(read_data);
}
}
private:
const FlatSearcher<BATCH_SIZE> *owner_{nullptr};
IndexStorage::Segment::Pointer features_segment_{};
uint32_t total_vector_count_{0};
uint32_t feature_size_{0};
std::vector<uint8_t> block_buffer_{};
const uint8_t *data_{nullptr};
uint64_t offset_{0};
uint32_t cursor_index_{0};
bool invalid_{false};
};
//! Retrieve a vector via local index
const void *get_vector_by_index(uint32_t index) const {
const void *read_data = nullptr;
if (index == kInvalidNodeId) {
LOG_ERROR("Failed to get vector by Invalid Id.");
return nullptr;
}
if (owner_->column_major_order() &&
index < (total_vector_count_ - (total_vector_count_ % BATCH_SIZE))) {
uint32_t block_size = feature_size_ * BATCH_SIZE;
uint64_t offset = (index - (index % BATCH_SIZE)) * feature_size_;
if (features_segment_->read(offset, &read_data, block_size) !=
block_size) {
LOG_ERROR("Failed to read data (%u bytes) from features segment",
block_size);
return nullptr;
}
uint32_t align_size = IndexMeta::AlignSizeof(owner_->meta().data_type());
ReverseTranspose<BATCH_SIZE>(
align_size, read_data, feature_size_ / align_size, &block_buffer_[0]);
read_data = block_buffer_.data() + ((index % BATCH_SIZE) * feature_size_);
} else {
if (features_segment_->read(index * feature_size_, &read_data,
feature_size_) != feature_size_) {
LOG_ERROR("Failed to read data (%u bytes) from features segment",
feature_size_);
return nullptr;
}
}
return read_data;
}
private:
//! Members
const FlatSearcher<BATCH_SIZE> *owner_{nullptr};
IndexStorage::Segment::Pointer features_segment_{};
uint32_t feature_size_{0};
uint32_t total_vector_count_{0};
mutable std::vector<uint8_t> block_buffer_{};
};
} // namespace core
} // namespace zvec
+491
View File
@@ -0,0 +1,491 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "flat_streamer.h"
#include <framework/index_factory.h>
#include "flat_streamer_context.h"
#include "flat_streamer_dumper.h"
#include "flat_streamer_provider.h"
namespace zvec {
namespace core {
#define WRITE_LOCK_GUARD(MUTEX, LOCK_NAME) \
ailego::WriteLock write_lock(MUTEX); \
std::unique_lock<ailego::WriteLock> LOCK_NAME(write_lock);
#define READ_LOCK_GUARD_DEFER(MUTEX, LOCK_NAME) \
ailego::ReadLock read_lock(MUTEX); \
std::unique_lock<ailego::ReadLock> LOCK_NAME(read_lock, std::defer_lock);
template <size_t BATCH_SIZE>
FlatStreamer<BATCH_SIZE>::FlatStreamer() : entity_(stats_) {}
template <size_t BATCH_SIZE>
FlatStreamer<BATCH_SIZE>::~FlatStreamer() {
if (state_ == STATE_INITED) {
this->cleanup();
}
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::init(const IndexMeta &imeta,
const ailego::Params &params) {
meta_ = imeta;
meta_.set_streamer("FlatStreamer", 0U, params);
int error_code = InitializeMetric(meta_, &metric_);
if (error_code != 0) {
LOG_ERROR("Failed to initialize index metric %s, error=%d, %s",
meta_.metric_name().c_str(), error_code,
IndexError::What(error_code));
return error_code;
}
if (metric_->query_metric()) {
metric_ = metric_->query_metric();
}
// 参数设置
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order_)) {
meta_.set_major_order(column_major_order_ ? IndexMeta::MO_COLUMN
: IndexMeta::MO_ROW);
}
// Verify column major order
if (meta_.major_order() != IndexMeta::MO_ROW) {
IndexMeta::DataType ft = meta_.data_type();
bool support_column_major = true;
if ((ft != IndexMeta::DT_FP32 && ft != IndexMeta::DT_FP16 &&
ft != IndexMeta::DT_INT8 && ft != IndexMeta::DT_INT4 &&
ft != IndexMeta::DT_BINARY32 && ft != IndexMeta::DT_BINARY64) ||
(meta_.unit_size() != IndexMeta::UnitSizeof(ft))) {
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
LOG_ERROR("Unsupported type %d with unit size %u.", ft,
meta_.unit_size());
return IndexError_Unsupported;
} else {
support_column_major = false;
}
}
if (meta_.element_size() % IndexMeta::AlignSizeof(ft) != 0) {
if (meta_.major_order() == IndexMeta::MO_COLUMN) {
LOG_ERROR("Unsupported type %d with dimension %u.", ft,
meta_.dimension());
return IndexError_Unsupported;
} else {
support_column_major = false;
}
}
if (meta_.major_order() == IndexMeta::MO_UNDEFINED &&
support_column_major) {
meta_.set_major_order(IndexMeta::MO_ROW);
}
}
if (!VerifyMetric(meta_)) {
LOG_ERROR("Invalid index metric %s.", meta_.metric_name().c_str());
return IndexError_InvalidArgument;
}
read_block_size_ = FLAT_READ_BLOCK_SIZE;
params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
params.get(FLAT_SEARCHER_USE_ID_MAP, &use_key_info_map_);
// entity init
uint32_t block_vector_count = kDefaultBlockVecCount;
uint32_t segment_size = kDefaultSegmentSize;
bool filter_same_key = true;
entity_.set_block_vector_count(block_vector_count);
entity_.set_segment_size(segment_size);
entity_.enable_filter_same_key(filter_same_key);
entity_.set_linear_list_count(1);
entity_.set_use_key_info_map(use_key_info_map_);
*entity_.mutable_meta() = meta_;
state_ = STATE_INITED;
return 0;
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::cleanup() {
if (state_ == STATE_OPENED) {
this->close();
}
LOG_DEBUG("FlatStreamer cleanup");
state_ = STATE_INIT;
return 0;
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::open(IndexStorage::Pointer stg) {
if (!stg) {
LOG_ERROR("Failed to open for invalid storage");
return IndexError_InvalidArgument;
}
if (ailego_unlikely(state_ != STATE_INITED)) {
LOG_ERROR("Open storage failed, init streamer first!");
return IndexError_NoReady;
}
LOG_DEBUG("FlatStreamer open with %s", stg->name().c_str());
int ret = entity_.open(std::move(stg), meta_);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Failed to open storage");
return ret;
}
magic_ = IndexContext::GenerateMagic();
state_ = STATE_OPENED;
return 0;
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::close(void) {
LOG_DEBUG("FlatStreamer close");
entity_.flush_linear_meta();
stats_.clear();
int ret = entity_.close();
if (ailego_unlikely(ret != 0)) {
return ret;
}
state_ = STATE_INITED;
return 0;
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::flush(uint64_t checkpoint) {
LOG_INFO("FlatStreamer flush with checkpoint %zu", (size_t)checkpoint);
return entity_.flush(checkpoint);
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::dump(const IndexDumper::Pointer &dumper) {
std::string searcher_name = "FlatSearcher";
if (BATCH_SIZE == 16) {
searcher_name = "FlatSearcher16";
}
meta_.set_searcher(searcher_name, 0U, ailego::Params());
WRITE_LOCK_GUARD(dump_mutex_, dump_lock);
std::shared_ptr<FlatStreamerDumper<BATCH_SIZE>> bf_dumper =
std::make_shared<FlatStreamerDumper<BATCH_SIZE>>(this);
int ret = bf_dumper->dump(dumper);
*(stats_.mutable_dumped_size()) += bf_dumper->dump_size();
return ret;
}
template <size_t BATCH_SIZE>
IndexStreamer::Context::UPointer FlatStreamer<BATCH_SIZE>::create_context(
void) const {
if (state_ != STATE_OPENED) {
LOG_ERROR("Failed to create Context, open storage first!");
return Context::UPointer();
}
return IndexStreamer::Context::Pointer(
new FlatStreamerContext<BATCH_SIZE>(this));
}
template <size_t BATCH_SIZE>
IndexProvider::Pointer FlatStreamer<BATCH_SIZE>::create_provider(void) const {
return IndexProvider::Pointer(new (std::nothrow)
FlatStreamerProvider<BATCH_SIZE>(this));
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::add_impl(uint64_t pkey, const void *query,
const IndexQueryMeta &qmeta,
Context::UPointer &context) {
if (!query || qmeta.dimension() != meta_.dimension() ||
qmeta.data_type() != meta_.data_type() ||
qmeta.element_size() != meta_.element_size()) {
LOG_ERROR(
"Failed to add for invalid arguments, query=%p, qmeta(type=%u "
"dim=%u size=%u) vs meta(type=%u dim=%u size=%u)",
query, qmeta.data_type(), qmeta.dimension(), qmeta.element_size(),
meta_.data_type(), meta_.dimension(), meta_.element_size());
(*stats_.mutable_discarded_count())++;
return IndexError_InvalidArgument;
}
auto *ctx = dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
if (!ctx) {
LOG_ERROR("Failed to cast FlatStreamerContext");
(*stats_.mutable_discarded_count())++;
return IndexError_Cast;
}
READ_LOCK_GUARD_DEFER(dump_mutex_, dump_lock);
if (!dump_lock.try_lock()) {
LOG_ERROR("Cannot add vector while dumping index");
(*stats_.mutable_discarded_count())++;
return IndexError_Unsupported;
}
// IndexQueryMeta iv_qmeta;
// int ret = entity_.convert(query, qmeta, &query, &iv_qmeta);
// if (ret != 0) {
// LOG_ERROR("Failed to convert record for %s",
// IndexError::What(ret));
// (*stats_.mutable_discarded_count())++;
// return ret;
// }
int ret = entity_.add(pkey, query, qmeta.element_size());
if (ret != 0) {
LOG_ERROR("Failed to add record for %s", IndexError::What(ret));
(*stats_.mutable_discarded_count())++;
return ret;
}
return 0;
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) {
if (!query || qmeta.dimension() != meta_.dimension() ||
qmeta.data_type() != meta_.data_type() ||
qmeta.element_size() != meta_.element_size()) {
LOG_ERROR(
"Failed to add for invalid arguments, query=%p, qmeta(type=%u "
"dim=%u size=%u) vs meta(type=%u dim=%u size=%u)",
query, qmeta.data_type(), qmeta.dimension(), qmeta.element_size(),
meta_.data_type(), meta_.dimension(), meta_.element_size());
(*stats_.mutable_discarded_count())++;
return IndexError_InvalidArgument;
}
auto *ctx = dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
if (!ctx) {
LOG_ERROR("Failed to cast FlatStreamerContext");
(*stats_.mutable_discarded_count())++;
return IndexError_Cast;
}
READ_LOCK_GUARD_DEFER(dump_mutex_, dump_lock);
if (!dump_lock.try_lock()) {
LOG_ERROR("Cannot add vector while dumping index");
(*stats_.mutable_discarded_count())++;
return IndexError_Unsupported;
}
int ret = entity_.add_vector_with_id(id, query, qmeta.element_size());
if (ret != 0) {
LOG_ERROR("Failed to add record for %s", IndexError::What(ret));
(*stats_.mutable_discarded_count())++;
return ret;
}
return 0;
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::search_bf_impl(const void *query,
const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const {
ailego_assert(query && count && !!context);
ailego_assert(metric_->is_matched(meta_, qmeta));
FlatStreamerContext<BATCH_SIZE> *bf_context =
dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
if (!bf_context) {
LOG_ERROR("Invalid brute-force streamer context");
return IndexError_InvalidArgument;
}
if (bf_context->magic() != magic_) {
bf_context->reset(this);
}
if (bf_context->group_by_search()) {
return group_by_search_impl(query, qmeta, count, context);
}
bf_context->reset_results(count);
auto &filter = bf_context->filter();
for (size_t q = 0; q < count; ++q) {
auto *heap = bf_context->result_heap();
auto *context_stats = bf_context->mutable_stats(q);
uint32_t scan_count = 0;
int ret = entity_.search(query, filter, &scan_count, heap, context_stats);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Failed to search for %s", IndexError::What(ret));
return ret;
}
heap->sort();
bf_context->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
return 0;
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
ailego_assert(query && count && !!context);
ailego_assert(metric_->is_matched(meta_, qmeta));
FlatStreamerContext<BATCH_SIZE> *bf_context =
dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
if (!bf_context) {
LOG_ERROR("Invalid brute-force streamer context");
return IndexError_InvalidArgument;
}
if (bf_context->magic() != magic_) {
bf_context->reset(this);
}
if (bf_context->group_by_search()) {
return group_by_search_p_keys_impl(query, p_keys, qmeta, count, context);
}
bf_context->reset_results(count);
auto &filter = bf_context->filter();
for (size_t q = 0; q < count; ++q) {
auto *heap = bf_context->result_heap();
for (node_id_t idx = 0; idx < p_keys[q].size(); ++idx) {
uint64_t key = p_keys[q][idx];
if (!filter.is_valid() || !filter(key)) {
dist_t dist = 0;
IndexStorage::MemoryBlock block;
entity_.get_vector_by_key(key, block);
entity_.row_major_distance(query, block.data(), 1, &dist);
heap->emplace(key, dist);
}
}
heap->sort();
bf_context->topk_to_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
return 0;
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::group_by_search_impl(
const void *query, const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
FlatStreamerContext<BATCH_SIZE> *bf_context =
dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
if (!bf_context) {
LOG_ERROR("Invalid brute-force streamer context");
return IndexError_InvalidArgument;
}
bf_context->resize_group_results(count);
if (!bf_context->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(uint64_t)> group_by = [&](uint64_t key) {
return bf_context->group_by()(key);
};
auto iterator = entity_.creater_iterator();
for (size_t q = 0; q < count; ++q) {
bf_context->group_topk_heaps().clear();
for (node_id_t id = 0; id < entity_.vector_count(); ++id) {
uint64_t key = entity_.key(id);
if (!bf_context->filter().is_valid() || !bf_context->filter()(key)) {
dist_t dist = 0;
IndexStorage::MemoryBlock block;
entity_.get_vector_by_key(key, block);
entity_.row_major_distance(query, block.data(), 1, &dist);
std::string group_id = group_by(key);
auto &topk_heap = bf_context->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(bf_context->group_topk());
}
topk_heap.emplace(key, dist);
}
}
bf_context->topk_to_group_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
return 0;
}
template <size_t BATCH_SIZE>
int FlatStreamer<BATCH_SIZE>::group_by_search_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const {
FlatStreamerContext<BATCH_SIZE> *bf_context =
dynamic_cast<FlatStreamerContext<BATCH_SIZE> *>(context.get());
if (!bf_context) {
LOG_ERROR("Invalid brute-force streamer context");
return IndexError_InvalidArgument;
}
bf_context->resize_group_results(count);
if (!bf_context->group_by().is_valid()) {
LOG_ERROR("Invalid group-by function");
return IndexError_InvalidArgument;
}
std::function<std::string(uint64_t)> group_by = [&](uint64_t key) {
return bf_context->group_by()(key);
};
auto iterator = entity_.creater_iterator();
for (size_t q = 0; q < count; ++q) {
bf_context->group_topk_heaps().clear();
for (node_id_t idx = 0; idx < p_keys[q].size(); ++idx) {
uint64_t key = p_keys[q][idx];
if (!bf_context->filter().is_valid() || !bf_context->filter()(key)) {
dist_t dist = 0;
IndexStorage::MemoryBlock block;
entity_.get_vector_by_key(key, block);
entity_.row_major_distance(query, block.data(), 1, &dist);
std::string group_id = group_by(key);
auto &topk_heap = bf_context->group_topk_heaps()[group_id];
if (topk_heap.empty()) {
topk_heap.limit(bf_context->group_topk());
}
topk_heap.emplace(key, dist);
}
}
bf_context->topk_to_group_result(q);
query = static_cast<const char *>(query) + qmeta.element_size();
}
return 0;
}
INDEX_FACTORY_REGISTER_STREAMER_ALIAS(LinearStreamer, FlatStreamer<32>);
INDEX_FACTORY_REGISTER_STREAMER_ALIAS(FlatStreamer, FlatStreamer<32>);
INDEX_FACTORY_REGISTER_STREAMER_ALIAS(FlatStreamer16, FlatStreamer<16>);
INDEX_FACTORY_REGISTER_STREAMER_ALIAS(FlatStreamer32, FlatStreamer<32>);
} // namespace core
} // namespace zvec
+190
View File
@@ -0,0 +1,190 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/parallel/lock.h>
#include <framework/index_streamer.h>
#include "flat_streamer_entity.h"
#include "flat_utility.h"
namespace zvec {
namespace core {
/*! Flat Streamer
*/
template <size_t BATCH_SIZE>
class FlatStreamer : public IndexStreamer {
public:
using ContextPointer = IndexStreamer::Context::UPointer;
FlatStreamer(void);
virtual ~FlatStreamer(void);
FlatStreamer(const FlatStreamer &streamer) = delete;
FlatStreamer &operator=(const FlatStreamer &streamer) = delete;
public:
//! Initialize Streamer
int init(const IndexMeta &, const ailego::Params &) override;
//! Cleanup Streamer
int cleanup(void) override;
//! Create a context
IndexStreamer::Context::UPointer create_context(void) const override;
//! Create a new iterator
IndexProvider::Pointer create_provider(void) const override;
//! Add a vector into index
int add_impl(uint64_t pkey, const void *query, const IndexQueryMeta &qmeta,
Context::UPointer &context) override;
int add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Similarity search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::UPointer &context) const override {
return search_bf_impl(query, qmeta, 1, context);
}
//! Similarity search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::UPointer &context) const override {
return search_bf_impl(query, qmeta, count, context);
}
//! Similarity brute force search
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::UPointer &context) const override {
return search_bf_impl(query, qmeta, 1, context);
}
//! Similarity brute force search
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::UPointer &context) const override;
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta,
Context::UPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::UPointer &context) const override;
int group_by_search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::UPointer &context) const;
int group_by_search_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
Context::Pointer &context) const;
//! Open index from file path
int open(IndexStorage::Pointer stg) override;
//! Close file
int close(void) override;
//! flush file
int flush(uint64_t checkpoint) override;
//! Dump index into storage
int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
const Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
const IndexMeta &meta(void) const override {
return meta_;
}
const FlatStreamerEntity &entity(void) const {
return entity_;
}
virtual const void *get_vector(uint64_t key) const override {
return this->get_vector_by_key(key);
}
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return this->get_vector_by_key(key, block);
}
const void *get_vector_by_key(uint64_t key) const {
return entity_.get_vector_by_key(key);
}
int get_vector_by_key(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_.get_vector_by_key(key, block);
}
const void *get_vector_by_id(uint32_t id) const override {
return get_vector_by_key(id);
}
int get_vector_by_id(const uint32_t id,
IndexStorage::MemoryBlock &block) const override {
return get_vector_by_key(id, block);
}
uint32_t magic(void) const {
return magic_;
}
//! Retrieve block size of data read
uint32_t read_block_size(void) const {
return read_block_size_;
}
private:
//! Constants
static constexpr uint32_t kDefaultBlockVecCount = 32u;
static constexpr uint32_t kDefaultSegmentSize = 4 * 1024 * 1024u;
static constexpr float kDefaultDocsSoftLimitRatio = 0.9f;
enum State { STATE_INIT = 0, STATE_INITED = 1, STATE_OPENED = 2 };
//! Members
uint32_t magic_{0};
uint32_t docs_hard_limit_{std::numeric_limits<uint32_t>::max()};
uint32_t docs_soft_limit_{0};
IndexMeta meta_{};
std::vector<std::vector<std::string>> data_;
IndexStreamer::Stats stats_{};
IndexMetric::Pointer metric_{};
State state_{STATE_INIT};
mutable std::mutex mapping_mutex_{};
ailego::SharedMutex dump_mutex_{};
FlatStreamerEntity entity_;
bool column_major_order_{false};
bool use_key_info_map_{true};
uint32_t read_block_size_{0};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,244 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "flat_streamer.h"
namespace zvec {
namespace core {
/*! Brute Force Streamer Context
*/
template <size_t BATCH_SIZE>
class FlatStreamerContext : public IndexStreamer::Context {
public:
//! Constructor
FlatStreamerContext(const FlatStreamer<BATCH_SIZE> *owner) {
this->reset(owner);
}
//! Destructor
virtual ~FlatStreamerContext(void) = default;
//! Set topk of search result
void set_topk(uint32_t topk) override {
topk_ = topk;
result_heap_.limit(topk);
}
//! Retrieve search result
const IndexDocumentList &result(void) const override {
return results_[0];
}
//! Retrieve search result with index
const IndexDocumentList &result(size_t idx) const override {
return results_[idx];
}
//! Retrieve result object for output
IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
inline IndexDocumentHeap *result_heap() {
return &result_heap_;
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(void) const override {
return group_results_[0];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(
size_t idx) const override {
return group_results_[idx];
}
//! Update the parameters of context
int update(const ailego::Params & /*params*/) override {
return 0;
}
//! Retrieve magic number
uint32_t magic(void) const override {
return magic_;
}
//! Get group topk
inline uint32_t group_topk() const {
return group_topk_;
}
//! Get group num
inline uint32_t group_num() const {
return group_num_;
}
inline std::map<std::string, TopkHeap> &group_topk_heaps() {
return group_topk_heaps_;
}
void set_fetch_vector(bool v) override {
fetch_vector_ = v;
}
bool fetch_vector() const override {
return fetch_vector_;
}
inline void resize_group_results(size_t size) {
if (group_by_search()) {
group_results_.resize(size);
}
}
void topk_to_result(uint32_t idx) {
if (ailego_unlikely(result_heap_.size() == 0)) {
return;
}
ailego_assert_with(idx < results_.size(), "invalid idx");
int size = std::min(topk_, static_cast<uint32_t>(result_heap_.size()));
result_heap_.sort();
results_[idx].clear();
for (int i = 0; i < size; ++i) {
auto score = result_heap_[i].score();
if (score > this->threshold()) {
break;
}
key_t key = result_heap_[i].key();
if (fetch_vector_) {
IndexStorage::MemoryBlock block;
owner_->entity().get_vector_by_key(key, block);
results_[idx].emplace_back(key, score, key, block);
} else {
results_[idx].emplace_back(key, score);
}
}
}
void topk_to_group_result(uint32_t idx) {
ailego_assert_with(idx < group_results_.size(), "invalid idx");
group_results_[idx].clear();
std::vector<std::pair<std::string, TopkHeap>> group_topk_list;
std::vector<std::pair<std::string, float>> best_score_in_groups;
for (auto itr = group_topk_heaps_.begin(); itr != group_topk_heaps_.end();
itr++) {
const std::string &group_id = (*itr).first;
auto &heap = (*itr).second;
heap.sort();
if (heap.size() > 0) {
float best_score = heap[0].second;
best_score_in_groups.push_back(std::make_pair(group_id, best_score));
}
}
std::sort(best_score_in_groups.begin(), best_score_in_groups.end(),
[](const std::pair<std::string, float> &a,
const std::pair<std::string, float> &b) -> int {
return a.second < b.second;
});
// truncate to group num
for (uint32_t i = 0; i < group_num() && i < best_score_in_groups.size();
++i) {
const std::string &group_id = best_score_in_groups[i].first;
group_topk_list.emplace_back(
std::make_pair(group_id, group_topk_heaps_[group_id]));
}
group_results_[idx].resize(group_topk_list.size());
for (uint32_t i = 0; i < group_topk_list.size(); ++i) {
const std::string &group_id = group_topk_list[i].first;
group_results_[idx][i].set_group_id(group_id);
uint32_t size = std::min(
group_topk_, static_cast<uint32_t>(group_topk_list[i].second.size()));
for (uint32_t j = 0; j < size; ++j) {
auto score = group_topk_list[i].second[j].second;
if (score > this->threshold()) {
break;
}
node_id_t id = group_topk_list[i].second[j].first;
auto provider = owner_->create_provider();
if (fetch_vector_) {
IndexStorage::MemoryBlock block;
provider->get_vector(id, block);
group_results_[idx][i].mutable_docs()->emplace_back(id, score, id,
block);
} else {
group_results_[idx][i].mutable_docs()->emplace_back(id, score, id);
}
}
}
}
//! Get if group by search
bool group_by_search() {
return group_num_ > 0;
}
//! Set group params
void set_group_params(uint32_t group_num, uint32_t group_topk) override {
group_num_ = group_num;
group_topk_ = group_topk;
group_topk_heaps_.clear();
}
void reset() override {}
//! Reset the context
void reset(const FlatStreamer<BATCH_SIZE> *owner) {
magic_ = owner->magic();
feature_size_ = owner->meta().element_size();
uint32_t block_size = feature_size_ * BATCH_SIZE;
actual_read_size_ =
(owner->read_block_size() + block_size - 1) / block_size * block_size;
owner_ = owner;
}
//! Reset all the query results
void reset_results(size_t qnum) {
results_.resize(qnum);
stats_vec_.resize(qnum);
for (size_t i = 0; i < qnum; ++i) {
results_[i].clear();
stats_vec_[i].clear();
}
result_heap_.clear();
result_heap_.limit(topk_);
result_heap_.set_threshold(this->threshold());
}
Stats *mutable_stats(size_t idx = 0) {
ailego_assert_with(stats_vec_.size() > idx, "invalid index");
return &stats_vec_[idx];
}
private:
const FlatStreamer<BATCH_SIZE> *owner_{nullptr};
std::vector<Stats> stats_vec_{};
uint32_t magic_{0};
uint32_t topk_{0};
uint32_t feature_size_{0};
uint32_t actual_read_size_{0};
IndexDocumentHeap result_heap_;
std::vector<IndexDocumentList> results_{};
std::string batch_queries_{};
float scores_[BATCH_SIZE * BATCH_SIZE];
bool fetch_vector_{false};
// group
uint32_t group_num_{0};
uint32_t group_topk_{0};
std::map<std::string, TopkHeap> group_topk_heaps_{};
std::vector<IndexGroupDocumentList> group_results_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,243 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "flat_streamer.h"
#include "flat_utility.h"
namespace zvec {
namespace core {
template <size_t BATCH_SIZE>
class FlatStreamerDumper {
public:
typedef std::unique_ptr<FlatStreamerDumper> Pointer;
FlatStreamerDumper(const FlatStreamer<BATCH_SIZE> *owner) {
owner_ = owner;
dump_size_ = 0;
}
int dump(const IndexDumper::Pointer &dumper) {
ailego::ElapsedTime stamp;
std::vector<uint64_t> keys;
if (owner_->meta().major_order() == IndexMeta::MO_COLUMN) {
int error_code = this->write_column_index(dumper.get(), &keys);
if (error_code != 0) {
return error_code;
}
} else {
int error_code = this->write_row_index(dumper.get(), &keys);
if (error_code != 0) {
return error_code;
}
}
int error_code = this->write_keys(keys, dumper.get());
if (error_code != 0) {
return error_code;
}
error_code = this->write_mapping(keys, dumper.get());
if (error_code != 0) {
return error_code;
}
error_code = IndexHelper::SerializeToDumper(owner_->meta(), dumper.get());
if (error_code != 0) {
return error_code;
}
LOG_DEBUG("dumped_count: %zu, costtime: %zu", keys.size(),
(size_t)stamp.milli_seconds());
return 0;
}
size_t dump_size() {
return dump_size_;
}
private:
int write_column_index(IndexDumper *dumper, std::vector<uint64_t> *keys) {
switch (IndexMeta::AlignSizeof(owner_->meta().data_type())) {
case 2:
return this->write_column_index<uint16_t>(dumper, keys);
case 4:
return this->write_column_index<uint32_t>(dumper, keys);
case 8:
return this->write_column_index<uint64_t>(dumper, keys);
default:
ailego_check_with(0, "BAD CASE");
}
return IndexError_Runtime;
}
template <typename T>
int write_column_index(IndexDumper *dumper, std::vector<uint64_t> *keys) {
auto iter = owner_->entity().creater_iterator();
if (!iter) {
LOG_ERROR("Failed to create iterator");
return IndexError_Runtime;
}
// Write features
size_t element_size = owner_->meta().element_size();
size_t block_size = element_size * BATCH_SIZE;
std::string block1, block2;
block1.reserve(block_size);
block2.reserve(block_size);
for (; iter->is_valid(); iter->next()) {
block1.append(reinterpret_cast<const char *>(iter->data()), element_size);
keys->emplace_back(iter->key());
if (block1.size() == block_size) {
ailego::MatrixHelper::Transpose<T, BATCH_SIZE>(
block1.data(), element_size / sizeof(T), (void *)block2.data());
if (dumper->write(block2.data(), block_size) != block_size) {
LOG_ERROR("Failed to write data into dumper %s",
dumper->name().c_str());
return IndexError_WriteData;
}
block1.clear();
dump_size_ += block_size;
}
}
if (!block1.empty()) {
if (dumper->write(block1.data(), block1.size()) != block1.size()) {
LOG_ERROR("Failed to write data into dumper %s",
dumper->name().c_str());
return IndexError_WriteData;
}
dump_size_ += block1.size();
}
// Write the padding if need
size_t features_size = keys->size() * element_size;
size_t features_padding_size =
ailego_align(features_size, 32) - features_size;
if (features_padding_size) {
std::string padding(features_padding_size, '\0');
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
LOG_ERROR("Failed to write data into dumper %s",
dumper->name().c_str());
return IndexError_WriteData;
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
features_padding_size, 0);
}
int write_row_index(IndexDumper *dumper, std::vector<uint64_t> *keys) {
auto iter = owner_->entity().creater_iterator();
if (!iter) {
LOG_ERROR("Failed to create iterator");
return IndexError_Runtime;
}
// Write features
size_t element_size = owner_->meta().element_size();
for (; iter->is_valid(); iter->next()) {
if (dumper->write(iter->data(), element_size) != element_size) {
LOG_ERROR("Failed to write data into dumper %s",
dumper->name().c_str());
return IndexError_WriteData;
}
dump_size_ += element_size;
keys->emplace_back(iter->key());
}
// Write the padding if need
size_t features_size = keys->size() * element_size;
size_t features_padding_size =
ailego_align(features_size, 32) - features_size;
if (features_padding_size) {
std::string padding(features_padding_size, '\0');
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
LOG_ERROR("Failed to write data into dumper %s",
dumper->name().c_str());
return IndexError_WriteData;
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
features_padding_size, 0);
}
int write_keys(const std::vector<uint64_t> &keys, IndexDumper *dumper) {
size_t keys_size = keys.size() * sizeof(uint64_t);
size_t keys_padding_size = ailego_align(keys_size, 32) - keys_size;
if (dumper->write(keys.data(), keys_size) != keys_size) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
dump_size_ += keys_size;
// Write the padding if need
if (keys_padding_size) {
std::string padding(keys_padding_size, '\0');
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
LOG_ERROR("Failed to write data into dumper %s",
dumper->name().c_str());
return IndexError_WriteData;
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
0);
}
int write_mapping(const std::vector<uint64_t> &keys, IndexDumper *dumper) {
std::vector<uint32_t> mapping(keys.size());
std::iota(mapping.begin(), mapping.end(), 0);
std::sort(mapping.begin(), mapping.end(),
[&keys](uint32_t lhs, uint32_t rhs) {
return (keys[lhs] < keys[rhs]);
});
size_t mapping_size = mapping.size() * sizeof(uint32_t);
size_t mapping_padding_size = ailego_align(mapping_size, 32) - mapping_size;
if (dumper->write(mapping.data(), mapping_size) != mapping_size) {
LOG_ERROR("Failed to write data into dumper %s", dumper->name().c_str());
return IndexError_WriteData;
}
dump_size_ += mapping_size;
// Write the padding if need
if (mapping_padding_size) {
std::string padding(mapping_padding_size, '\0');
if (dumper->write(padding.data(), padding.size()) != padding.size()) {
LOG_ERROR("Failed to write data into dumper %s",
dumper->name().c_str());
return IndexError_WriteData;
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
mapping_padding_size, 0);
}
private:
const FlatStreamer<BATCH_SIZE> *owner_{nullptr};
size_t dump_size_{0};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,994 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "flat_streamer_entity.h"
#include <cstdint>
#include "flat_utility.h"
#include "index_error.h"
namespace zvec {
namespace core {
FlatStreamerEntity::FlatStreamerEntity(IndexStreamer::Stats &stats)
: stats_(stats) {}
int FlatStreamerEntity::open(IndexStorage::Pointer storage,
const IndexMeta & /*mt*/) {
if (storage_) {
LOG_ERROR("An storage instance is already opened");
return IndexError_Duplicate;
}
// segments_[0] store the meta information of the linear list
ailego_assert_with(segments_.size() == 0, "Invalid Size");
key_info_map_lock_ = std::make_shared<ailego::SharedMutex>();
key_info_map_.clear();
id_key_vector_.clear();
withid_key_info_map_.clear();
withid_key_map_.clear();
vec_unit_size_ = IndexMeta::AlignSizeof(index_meta_.data_type());
vec_cols_ = index_meta_.element_size() / vec_unit_size_;
meta_.header.block_size =
ailego_align(sizeof(BlockHeader) + sizeof(DeletionMap) +
(index_meta_.element_size() + sizeof(uint64_t)) *
meta_.header.block_vector_count,
32);
if (storage->get(FLAT_LINEAR_LIST_HEAD_SEG_ID) ||
storage->get(FLAT_LINEAR_META_SEG_ID)) {
int ret = this->load_storage(storage);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Failed to load storage index");
return ret;
}
} else {
int ret = this->init_storage(storage);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Failed to init storage");
return ret;
}
}
storage_ = storage;
//! Create the distance calculator
auto metric = IndexFactory::CreateMetric(index_meta_.metric_name());
if (!metric) {
LOG_ERROR("Failed to create metric %s", index_meta_.metric_name().c_str());
return IndexError_NoExist;
}
int ret = metric->init(index_meta_, index_meta_.metric_params());
if (ret != 0) {
LOG_ERROR("Failed to initialize metric %s",
index_meta_.metric_name().c_str());
return ret;
}
row_distance_ = metric->distance();
column_distance_ =
metric->distance_matrix(meta_.header.block_vector_count, 1);
LOG_DEBUG("Open storage %s done, metric=%s", storage_->name().c_str(),
index_meta_.metric_name().c_str());
return 0;
}
int FlatStreamerEntity::close(void) {
segments_.clear();
storage_.reset();
key_info_map_lock_.reset();
key_info_map_.clear();
withid_key_info_map_.clear();
withid_key_map_.clear();
id_key_vector_.clear();
meta_.create_time = 0;
meta_.update_time = 0;
meta_.segment_count = 0;
meta_.header.total_vector_count = 0;
meta_.header.block_count = 0;
meta_.header.block_size = 0;
meta_.header.linear_body_size = 0;
return 0;
}
int FlatStreamerEntity::flush_linear_meta(void) {
if (!storage_) {
return 0;
}
meta_.update_time = ailego::Realtime::Seconds();
meta_.revision_id = stats_.revision_id();
stats_.set_update_time(meta_.update_time);
auto segment = storage_->get(FLAT_LINEAR_META_SEG_ID);
if (ailego_unlikely(!segment)) {
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
return IndexError_Runtime;
}
if (segment->write(0, &meta_, sizeof(meta_)) != sizeof(meta_)) {
LOG_ERROR("Failed to write segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
return IndexError_WriteData;
}
return 0;
}
int FlatStreamerEntity::flush(uint64_t checkpoint) {
int ret = this->flush_linear_meta();
if (ailego_unlikely(ret != 0)) {
return ret;
}
if (checkpoint != 0) {
storage_->refresh(checkpoint);
}
ret = storage_->flush();
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Failed to refresh storage for %s", IndexError::What(ret));
return ret;
}
if (checkpoint != 0) {
stats_.set_check_point(checkpoint);
}
return 0;
}
int FlatStreamerEntity::add(uint64_t key, const void *vec, size_t size) {
std::lock_guard<std::mutex> lock(mutex_);
if (filter_same_key_) {
key_info_map_lock_->lock_shared();
if (key_info_map_.find(key) != key_info_map_.end()) {
key_info_map_lock_->unlock_shared();
LOG_WARN("Try to add duplicate key, drop it");
return IndexError_Duplicate;
}
key_info_map_lock_->unlock_shared();
}
if (size != static_cast<size_t>(index_meta_.element_size())) {
LOG_ERROR("Failed to add, mismatch size %zu vs elemsize %u", size,
index_meta_.element_size());
return IndexError_Mismatch;
}
IndexStorage::MemoryBlock head_block;
this->get_head_block(head_block);
const BlockLocation *bl =
reinterpret_cast<const BlockLocation *>(head_block.data());
if (ailego_unlikely(bl == nullptr)) {
LOG_ERROR("Failed to get block loc");
return IndexError_ReadData;
}
BlockLocation block = *bl;
if (!this->is_valid_block(block)) {
int ret = this->alloc_block(block, &block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
ret = this->update_head_block(block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
}
int ret = this->add_to_block(block, key, vec, size);
if (ret == IndexError_IndexFull) {
ret = this->alloc_block(block, &block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
ret = this->update_head_block(block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
ret = this->add_to_block(block, key, vec, size);
if (ailego_unlikely(ret != 0)) {
return ret;
}
}
if (ailego_unlikely(ret != 0)) {
return ret;
}
(*stats_.mutable_added_count())++;
stats_.set_revision_id(meta_.revision_id + 1);
return 0;
}
int FlatStreamerEntity::search(const void *query, const IndexFilter &filter,
uint32_t *scan_count, IndexDocumentHeap *heap,
IndexContext::Stats *context_stats) const {
IndexStorage::MemoryBlock head_block;
this->get_head_block(head_block);
const BlockLocation *bl =
reinterpret_cast<const BlockLocation *>(head_block.data());
if (ailego_unlikely(bl == nullptr)) {
LOG_ERROR("Failed to get block loc");
return IndexError_ReadData;
}
BlockLocation block = *bl;
while (this->is_valid_block(block)) {
IndexStorage::MemoryBlock block_header_block;
this->get_block_header(block, block_header_block);
const BlockHeader *hd =
reinterpret_cast<const BlockHeader *>(block_header_block.data());
if (ailego_unlikely(hd == nullptr)) {
LOG_ERROR("Failed to get block header");
return IndexError_ReadData;
}
if (hd->vector_count > 0) {
*scan_count += hd->vector_count;
IndexStorage::MemoryBlock deletion_map_block;
this->get_block_deletion_map(block, deletion_map_block);
const DeletionMap *deletion_map =
reinterpret_cast<const DeletionMap *>(deletion_map_block.data());
if (filter.is_valid() || deletion_map->is_dirty()) {
this->search_block(query, block, hd, 1.0, filter, deletion_map, heap,
context_stats);
} else {
*(context_stats->mutable_dist_calced_count()) += hd->vector_count;
this->search_block(query, block, hd, 1.0, heap);
}
}
block = hd->next;
}
return 0;
}
//! Search in a block
void FlatStreamerEntity::search_block(const void *query,
const BlockLocation &bl,
const BlockHeader *hd, float norm_val,
IndexDocumentHeap *heap) const {
std::vector<float> distances(block_vector_count());
IndexStorage::MemoryBlock vecs_block;
this->get_block_vectors(bl, vecs_block);
const char *vecs = reinterpret_cast<const char *>(vecs_block.data());
IndexStorage::MemoryBlock keys_block;
this->get_block_keys(bl, keys_block);
const uint64_t *keys = reinterpret_cast<const uint64_t *>(keys_block.data());
row_major_distance(query, vecs, hd->vector_count, distances.data());
for (size_t k = 0; k < hd->vector_count; ++k) {
if (keys[k] != kInvalidKey) {
heap->emplace(keys[k], distances[k] * norm_val);
}
}
}
//! Search in a block with filter
void FlatStreamerEntity::search_block(
const void *query, const BlockLocation &bl, const BlockHeader *hd,
float norm_val, const IndexFilter &filter, const DeletionMap *deletion_map,
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const {
std::vector<float> distances(block_vector_count());
IndexStorage::MemoryBlock vecs_block;
this->get_block_vectors(bl, vecs_block);
const char *vecs = reinterpret_cast<const char *>(vecs_block.data());
IndexStorage::MemoryBlock keys_block;
this->get_block_keys(bl, keys_block);
const uint64_t *keys = reinterpret_cast<const uint64_t *>(keys_block.data());
DeletionMap keeps;
for (size_t k = 0; k < hd->vector_count; ++k) {
const bool condition1 = !deletion_map->test(k);
const bool condition2 = filter.is_valid() ? !filter(keys[k]) : true;
const bool condition3 = keys[k] != kInvalidKey;
if (condition1 && condition2 && condition3) {
keeps.set(k);
}
}
if (!keeps.is_dirty()) {
(*context_stats->mutable_filtered_count()) += hd->vector_count;
return;
}
for (size_t k = 0; k < hd->vector_count; ++k) {
if (keeps.test(k)) {
auto cur_vec = vecs + index_meta_.element_size() * k;
row_major_distance(query, cur_vec, 1, distances.data() + k);
++(*context_stats->mutable_dist_calced_count());
}
}
for (size_t k = 0; k < hd->vector_count; ++k) {
if (keeps.test(k)) {
heap->emplace(keys[k], distances[k] * norm_val);
} else {
++(*context_stats->mutable_filtered_count());
}
}
}
int FlatStreamerEntity::search_bf(const void *query, const IndexFilter &filter,
IndexDocumentHeap *heap,
IndexContext::Stats *context_stats) const {
uint32_t scan_count;
return this->search(query, filter, &scan_count, heap, context_stats);
}
FlatStreamerEntity::Pointer FlatStreamerEntity::clone(void) const {
std::vector<IndexStorage::Segment::Pointer> segments;
segments.reserve(segments_.size());
for (size_t i = 0; i < segments_.size(); ++i) {
segments.emplace_back(segments_[i]->clone());
if (!segments[i]) {
LOG_ERROR("Failed to clone segment, index=%zu", i);
return nullptr;
}
}
auto entity = new (std::nothrow) FlatStreamerEntity(stats_);
if (!entity) {
LOG_ERROR("Failed to New FlatStreamerEntity object");
return nullptr;
}
entity->index_meta_ = this->index_meta_;
entity->storage_ = this->storage_;
// entity->reformer_ = this->reformer_;
entity->segments_ = segments;
entity->meta_ = this->meta_;
entity->key_info_map_lock_ = this->key_info_map_lock_;
entity->key_info_map_ = this->key_info_map_;
entity->id_key_vector_ = this->id_key_vector_;
entity->withid_key_info_map_ = this->withid_key_info_map_;
entity->withid_key_map_ = this->withid_key_map_;
entity->filter_same_key_ = this->filter_same_key_;
entity->vec_unit_size_ = this->vec_unit_size_;
entity->vec_cols_ = this->vec_cols_;
return FlatStreamerEntity::Pointer(entity);
}
const void *FlatStreamerEntity::get_vector_by_key(uint64_t key) const {
VectorLocation loc{};
key_info_map_lock_->lock_shared();
if (use_key_info_map_) {
auto iterator = key_info_map_.find(key);
if (iterator == key_info_map_.end()) {
key_info_map_lock_->unlock_shared();
return nullptr;
}
loc = iterator->second;
} else {
if (key < withid_key_info_map_.size()) {
loc = withid_key_info_map_[key];
} else {
key_info_map_lock_->unlock_shared();
return nullptr;
}
}
key_info_map_lock_->unlock_shared();
auto segment = this->get_segment(loc.segment_id);
const void *data = nullptr;
if (segment->read(loc.offset, &data, index_meta_.element_size()) !=
index_meta_.element_size()) {
LOG_ERROR("Failed to read segment, size=%u", index_meta_.element_size());
return nullptr;
}
return data;
}
int FlatStreamerEntity::get_vector_by_key(
const uint64_t key, IndexStorage::MemoryBlock &block) const {
VectorLocation loc{};
key_info_map_lock_->lock_shared();
if (use_key_info_map_) {
auto iterator = key_info_map_.find(key);
if (iterator == key_info_map_.end()) {
key_info_map_lock_->unlock_shared();
return -1;
}
loc = iterator->second;
} else {
if (key < withid_key_info_map_.size()) {
loc = withid_key_info_map_[key];
} else {
key_info_map_lock_->unlock_shared();
return -1;
}
}
key_info_map_lock_->unlock_shared();
auto segment = this->get_segment(loc.segment_id);
if (segment->read(loc.offset, block, index_meta_.element_size()) !=
index_meta_.element_size()) {
LOG_ERROR("Failed to read segment, size=%u", index_meta_.element_size());
return -1;
}
return 0;
}
IndexProvider::Iterator::Pointer FlatStreamerEntity::creater_iterator(
void) const {
auto entity = this->clone();
if (!entity) {
LOG_ERROR("Failed to clone entity");
return nullptr;
}
return Iterator::Pointer(new (std::nothrow)
FlatStreamerEntity::Iterator(std::move(entity)));
}
void FlatStreamerEntity::Iterator::read_next_block(void) {
auto block_size = entity_->linear_block_size();
while (segment_id_ < entity_->segments_.size()) {
auto &segment = entity_->segments_[segment_id_];
size_t off = block_index_ * block_size;
if (off + block_size > segment->data_size()) {
++segment_id_;
block_index_ = 0;
continue;
}
if (segment->read(off, block_, block_size) != block_size) {
LOG_ERROR("Failed to read block, off=%zu", off);
break;
}
data_ = block_.data();
auto hd = reinterpret_cast<const BlockHeader *>(
static_cast<const char *>(data_) + block_size - sizeof(BlockHeader));
if (hd->vector_count == 0) {
++block_index_;
continue;
}
block_vector_count_ = hd->vector_count;
block_vector_index_ = 0;
size_t elemsize = entity_->index_meta_.element_size();
keys_ = reinterpret_cast<const uint64_t *>(
reinterpret_cast<const char *>(data_) +
elemsize * entity_->block_vector_count());
return;
}
is_valid_ = false;
}
int FlatStreamerEntity::init_storage(IndexStorage::Pointer storage) {
// Init Linear Meta Segment
meta_.create_time = ailego::Realtime::Seconds();
stats_.set_create_time(meta_.create_time);
meta_.update_time = ailego::Realtime::Seconds();
stats_.set_update_time(meta_.update_time);
meta_.segment_count = 0;
meta_.revision_id = 0;
std::string str;
index_meta_.serialize(&str);
const size_t page = ailego::MemoryHelper::PageSize();
meta_.header.header_size = sizeof(LinearIndexHeader) + str.size();
meta_.header.total_vector_count = 0;
meta_.header.linear_body_size = 0;
meta_.header.block_count = 0;
meta_.header.index_meta_size = str.size();
meta_.header.linear_list_count = 1;
AdjustSegmentSize(&meta_);
LOG_DEBUG(
"Create Streamer Index, VecSize=%u, BlockSize=%u SegmentSize=%u "
"LinearListCount=%u",
index_meta_.element_size(), meta_.header.block_size, meta_.segment_size,
meta_.header.linear_list_count);
size_t size = ailego_align(sizeof(meta_) + str.size(), page);
int ret = storage->append(FLAT_LINEAR_META_SEG_ID, size);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Failed to append segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
return ret;
}
auto segment = storage->get(FLAT_LINEAR_META_SEG_ID);
if (ailego_unlikely(!segment)) {
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
return IndexError_Runtime;
}
if (segment->write(0, &meta_, sizeof(meta_)) != sizeof(meta_)) {
LOG_ERROR("Failed to write segment data");
return IndexError_WriteData;
}
if (segment->write(sizeof(meta_), str.data(), str.size()) != str.size()) {
LOG_ERROR("Failed to write segment data, size=%zu", str.size());
return IndexError_WriteData;
}
ret = storage->append("IndexMeta", str.size());
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Failed to append segment IndexMeta, code: %d", ret);
return ret;
}
auto index_meta_segment = storage->get("IndexMeta");
if (index_meta_segment->write(0, str.data(), str.size()) != str.size()) {
LOG_ERROR("Failed to write segment data, size=%zu", str.size());
return IndexError_WriteData;
}
*stats_.mutable_index_size() += size;
// Init Linear List Head Segment
size = ailego_align(sizeof(BlockLocation) * linear_list_count(), page);
ret = storage->append(FLAT_LINEAR_LIST_HEAD_SEG_ID, size);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Failed to append segment %s for %s, size=%zu",
FLAT_LINEAR_LIST_HEAD_SEG_ID.c_str(), IndexError::What(ret),
size);
return ret;
}
segment = storage->get(FLAT_LINEAR_LIST_HEAD_SEG_ID);
if (ailego_unlikely(!segment)) {
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_LIST_HEAD_SEG_ID.c_str());
return IndexError_Runtime;
}
if (segment->resize(size) != size) {
LOG_ERROR("Failed to resize segment, size=%zu", size);
return IndexError_WriteData;
}
segments_.emplace_back(std::move(segment));
*stats_.mutable_index_size() += size;
return 0;
}
int FlatStreamerEntity::load_linear_meta(IndexStorage::Pointer storage) {
AdjustSegmentSize(&meta_);
// Load Meta Segment
auto segment = storage->get(FLAT_LINEAR_META_SEG_ID);
if (!segment || segment->data_size() < sizeof(meta_)) {
LOG_ERROR("Missing segment %s, or invalid segment size",
FLAT_LINEAR_META_SEG_ID.c_str());
return IndexError_InvalidFormat;
}
IndexStorage::MemoryBlock data_block;
if (segment->read(0, data_block, segment->data_size()) !=
segment->data_size()) {
LOG_ERROR("Failed to read storage, size=%zu", segment->data_size());
return IndexError_InvalidFormat;
}
auto *mt = reinterpret_cast<const decltype(meta_) *>(data_block.data());
if (mt->header.block_vector_count != meta_.header.block_vector_count) {
LOG_ERROR("Unmatched BlockVecCount Setting, Index %u vs Setting %u",
mt->header.block_vector_count, meta_.header.block_vector_count);
return IndexError_Mismatch;
}
if (mt->header.block_size != meta_.header.block_size) {
LOG_ERROR("Unmatched BlockSize Setting, Index %u vs Setting %u",
mt->header.block_size, meta_.header.block_size);
return IndexError_Mismatch;
}
if (mt->header.index_meta_size + sizeof(meta_) > segment->data_size()) {
LOG_ERROR("Invalid format, IndexMetaSize %u, SegmentSize %zu",
mt->header.index_meta_size, segment->data_size());
return IndexError_InvalidFormat;
}
if (mt->header.linear_list_count != meta_.header.linear_list_count) {
LOG_ERROR("Unmatch LinearListCount, Index size %u vs Setting %u",
mt->header.linear_list_count, meta_.header.linear_list_count);
return IndexError_InvalidFormat;
}
IndexMeta index_meta;
if (!index_meta.deserialize(mt->header.index_meta,
mt->header.index_meta_size)) {
LOG_ERROR("Failed to deserialize IndexMeta, size=%u",
mt->header.index_meta_size);
return IndexError_InvalidFormat;
}
if (index_meta.data_type() != index_meta_.data_type() ||
index_meta.dimension() != index_meta_.dimension() ||
index_meta.element_size() != index_meta_.element_size() ||
index_meta.metric_name() != index_meta_.metric_name()) {
LOG_ERROR(
"Unmatch IndexMeta, Index(type=%u dim=%u elemsize=%u "
"metric=%s) Setting(type=%u dim=%u elemsize=%u metric=%s)",
index_meta.data_type(), index_meta.dimension(),
index_meta.element_size(), index_meta.metric_name().c_str(),
index_meta_.data_type(), index_meta_.dimension(),
index_meta_.element_size(), index_meta_.metric_name().c_str());
return IndexError_Mismatch;
}
// Segment Size can be reconfigurable
auto segment_size = meta_.segment_size;
std::memcpy(&meta_, mt, sizeof(meta_));
meta_.segment_size = segment_size;
return 0;
}
int FlatStreamerEntity::load_segment_keys_to_map(BlockLocation block) {
while (this->is_valid_block(block)) {
auto segment = this->get_segment(block.segment_id);
IndexStorage::MemoryBlock block_header_block;
this->get_block_header(block, block_header_block);
const BlockHeader *hd =
reinterpret_cast<const BlockHeader *>(block_header_block.data());
if (ailego_unlikely(hd == nullptr)) {
LOG_ERROR("Failed to get block header");
return IndexError_ReadData;
}
IndexStorage::MemoryBlock keys_block;
this->get_block_keys(block, keys_block);
const uint64_t *keys =
reinterpret_cast<const uint64_t *>(keys_block.data());
IndexStorage::MemoryBlock deletion_map_block;
this->get_block_deletion_map(block, deletion_map_block);
const DeletionMap *deletion_map =
reinterpret_cast<const DeletionMap *>(deletion_map_block.data());
for (uint32_t vector_index = 0; vector_index < hd->vector_count;
++vector_index) {
if (deletion_map->test(vector_index)) {
continue;
}
size_t vector_off =
this->get_block_vector_offset(block.block_index, vector_index);
key_info_map_[keys[vector_index]] =
VectorLocation(block.segment_id, false, vector_off);
id_key_vector_.push_back(keys[vector_index]);
}
block = hd->next;
}
return 0;
}
int FlatStreamerEntity::load_segment_keys_to_vector() {
for (uint32_t i = 0; i < meta_.header.total_vector_count; i++) {
size_t block_id = i / block_vector_count();
uint32_t vector_index = i % block_vector_count();
ailego_assert(segments_.size() > 1);
size_t segment_block_count =
segments_[1]->data_size() / linear_block_size();
size_t segment_id = block_id / segment_block_count + 1;
size_t real_block_id = block_id % segment_block_count;
size_t vector_off =
this->get_block_vector_offset(real_block_id, vector_index);
withid_key_info_map_.push_back(
VectorLocation(segment_id, false, vector_off));
size_t key_off = get_block_key_offset(real_block_id, vector_index);
withid_key_map_.push_back(key_off);
}
return 0;
}
int FlatStreamerEntity::load_storage(IndexStorage::Pointer storage) {
int ret = this->load_linear_meta(storage);
if (ailego_unlikely(ret != 0)) {
return ret;
}
// Load Linear List
auto hd_segment = storage->get(FLAT_LINEAR_LIST_HEAD_SEG_ID);
if (ailego_unlikely(!hd_segment)) {
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_LIST_HEAD_SEG_ID.c_str());
return IndexError_Runtime;
}
if (hd_segment->data_size() < linear_list_count() * sizeof(BlockLocation)) {
LOG_ERROR("Invalid segment size, LinearListCount=%zu, size=%zu",
linear_list_count(), hd_segment->data_size());
return IndexError_InvalidFormat;
}
segments_.emplace_back(hd_segment);
size_t index_size = hd_segment->capacity();
for (size_t i = 1; i <= meta_.segment_count; ++i) {
std::string segment_id =
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
auto seg = storage->get(segment_id);
if (!seg || seg->data_size() < meta_.header.block_size) {
LOG_ERROR("Failed to get segment %s, or invalid segment size",
segment_id.c_str());
return IndexError_InvalidFormat;
}
index_size += seg->capacity();
segments_.emplace_back(std::move(seg));
}
for (size_t i = 0; i < linear_list_count(); i++) {
IndexStorage::MemoryBlock head_block;
this->get_head_block(head_block);
const BlockLocation *bl =
reinterpret_cast<const BlockLocation *>(head_block.data());
if (ailego_unlikely(bl == nullptr)) {
LOG_ERROR("Failed to get block loc");
return IndexError_ReadData;
}
BlockLocation block = *bl;
if (use_key_info_map_) {
ret = this->load_segment_keys_to_map(block);
} else {
ret = this->load_segment_keys_to_vector();
}
if (ailego_unlikely(ret != 0)) {
return ret;
}
}
char create_time[32];
char update_time[32];
ailego::Realtime::Gmtime(meta_.create_time, "%Y-%m-%d %H:%M:%S", create_time,
sizeof(create_time));
ailego::Realtime::Gmtime(meta_.update_time, "%Y-%m-%d %H:%M:%S", update_time,
sizeof(update_time));
LOG_DEBUG(
"Load Index, IndexSize=%zu SegmentCount=%u SegmentSize=%u "
"RevisionId=%zu BlockCount=%u BlockSize=%u "
"BlockVectorCount=%u LinearListCount=%u TotalVecCount=%zu "
"CreateTime=%s UpdateTime=%s",
index_size, meta_.segment_count, meta_.segment_size,
static_cast<size_t>(meta_.revision_id), meta_.header.block_count,
meta_.header.block_size, meta_.header.block_vector_count,
meta_.header.linear_list_count,
static_cast<size_t>(meta_.header.total_vector_count), create_time,
update_time);
stats_.set_index_size(index_size);
stats_.set_check_point(storage->check_point());
stats_.set_create_time(meta_.create_time);
stats_.set_revision_id(meta_.revision_id);
stats_.set_update_time(meta_.update_time);
stats_.set_loaded_count(meta_.header.total_vector_count);
return 0;
}
int FlatStreamerEntity::alloc_segment(void) {
size_t index = segments_.size();
if (index == kMaxSegmentId) {
LOG_ERROR("Failed to alloc new segment, exceed max count %zu",
kMaxSegmentId);
return IndexError_IndexFull;
}
std::string segment_id =
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, index);
size_t size =
ailego_align(meta_.segment_size, ailego::MemoryHelper::PageSize());
auto segment = storage_->get(segment_id);
if (segment) {
if (segment->padding_size() < linear_block_size()) {
LOG_ERROR(
"Unexpect segment, index=%zu, data_size=%zu "
"padding_size=%zu block_size=%zu",
index, segment->data_size(), segment->padding_size(),
linear_block_size());
return IndexError_Runtime;
}
LOG_WARN("Alloc an existing segment=%s capacity=%zu", segment_id.c_str(),
segment->capacity());
} else {
int ret = storage_->append(segment_id, size);
if (ailego_unlikely(ret != 0)) {
LOG_ERROR("Failed to alloc segment from storage");
return ret;
}
segment = storage_->get(segment_id);
if (ailego_unlikely(!segment)) {
LOG_ERROR("Failed to get segment %s", segment_id.c_str());
return IndexError_Runtime;
}
}
meta_.segment_count += 1;
meta_.header.linear_body_size += size;
segments_.emplace_back(std::move(segment));
*stats_.mutable_index_size() += size;
// Update meta information
auto meta_segment = storage_->get(FLAT_LINEAR_META_SEG_ID);
if (ailego_unlikely(!meta_segment)) {
LOG_ERROR("Failed to get segment %s", FLAT_LINEAR_META_SEG_ID.c_str());
return IndexError_Runtime;
}
if (meta_segment->write(0, &meta_, sizeof(meta_)) != sizeof(meta_)) {
LOG_ERROR("Failed to write meta segment");
return IndexError_WriteData;
}
return 0;
}
int FlatStreamerEntity::alloc_block(const BlockLocation &next,
BlockLocation *block) {
if (segments_.size() <= 1 ||
segments_.back()->padding_size() < linear_block_size()) {
int ret = this->alloc_segment();
if (ailego_unlikely(ret != 0)) {
return ret;
}
}
auto &segment = segments_.back();
size_t block_index = segment->data_size() / linear_block_size();
if (block_index == kMaxBlockId) {
LOG_ERROR("Failed to alloc block, exceed max count %zu per segment",
kMaxBlockId);
return IndexError_IndexFull;
}
BlockHeader header;
header.next = next;
header.vector_count = 0;
header.column_major = false;
size_t hd_off = segment->data_size() + linear_block_size() - sizeof(header);
if (segment->write(hd_off, &header, sizeof(header)) != sizeof(header)) {
LOG_ERROR("Failed to write block header");
return IndexError_WriteData;
}
size_t del_off = hd_off - sizeof(DeletionMap);
DeletionMap reset_del_map{};
if (segment->write(del_off, &reset_del_map, sizeof(reset_del_map)) !=
sizeof(reset_del_map)) {
LOG_ERROR("Failed to write block deletion map");
return IndexError_WriteData;
}
++meta_.header.block_count;
block->segment_id = segments_.size() - 1;
block->block_index = (segment->data_size() / linear_block_size()) - 1;
return 0;
}
int FlatStreamerEntity::add_to_block(const BlockLocation &block, uint64_t key,
const void *data, size_t size) {
IndexStorage::MemoryBlock block_header_block;
this->get_block_header(block, block_header_block);
const BlockHeader *header =
reinterpret_cast<const BlockHeader *>(block_header_block.data());
if (ailego_unlikely(header == nullptr)) {
LOG_ERROR("Failed to get header");
return IndexError_ReadData;
}
if (header->vector_count == block_vector_count()) {
return IndexError_IndexFull;
}
auto &segment = segments_[block.segment_id];
size_t vector_off =
get_block_vector_offset(block.block_index, header->vector_count);
if (segment->write(vector_off, data, size) != size) {
LOG_ERROR("Failed to write vector, off=%zu size=%zu", vector_off, size);
return IndexError_WriteData;
}
size_t key_off =
get_block_key_offset(block.block_index, header->vector_count);
if (segment->write(key_off, &key, sizeof(key)) != sizeof(key)) {
LOG_ERROR("Failed to write key, off=%zu", key_off);
return IndexError_WriteData;
}
BlockHeader hd = *header;
hd.vector_count += 1;
size_t hd_off = get_block_header_offset(block.block_index);
if (segment->write(hd_off, &hd, sizeof(hd)) != sizeof(hd)) {
LOG_ERROR("Failed to write block header, off=%zu", hd_off);
return IndexError_WriteData;
}
VectorLocation loc(block.segment_id, false, vector_off);
key_info_map_lock_->lock();
key_info_map_[key] = loc;
id_key_vector_.push_back(key);
withid_key_info_map_.push_back(loc);
withid_key_map_.push_back(key_off);
key_info_map_lock_->unlock();
++meta_.header.total_vector_count;
return 0;
}
int FlatStreamerEntity::add_vector_with_id(const uint32_t id, const void *query,
const uint32_t size) {
std::lock_guard<std::mutex> lock(mutex_);
// if (filter_same_key_) {
// key_info_map_lock_->lock_shared();
// if (key_info_map_.find(id) != key_info_map_.end()) {
// key_info_map_lock_->unlock_shared();
// LOG_WARN("Try to add duplicate key, drop it");
// return IndexError_Duplicate;
// }
// key_info_map_lock_->unlock_shared();
// }
if (size != static_cast<size_t>(index_meta_.element_size())) {
LOG_ERROR("Failed to add, mismatch size %u vs elemsize %u", size,
index_meta_.element_size());
return IndexError_Mismatch;
}
if (id >= vector_count()) {
IndexStorage::MemoryBlock head_block;
this->get_head_block(head_block);
BlockLocation block =
*reinterpret_cast<const BlockLocation *>(head_block.data());
if (!this->is_valid_block(block)) {
int ret = this->alloc_block(block, &block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
ret = this->update_head_block(block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
}
for (size_t start_id = vector_count(); start_id < id; ++start_id) {
std::vector<char> vec(size);
int ret = this->add_to_block(block, kInvalidKey, vec.data(), size);
if (ret == IndexError_IndexFull) {
ret = this->alloc_block(block, &block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
ret = this->update_head_block(block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
ret = this->add_to_block(block, kInvalidKey, vec.data(), size);
if (ailego_unlikely(ret != 0)) {
return ret;
}
}
}
int ret = this->add_to_block(block, id, query, size);
if (ret == IndexError_IndexFull) {
ret = this->alloc_block(block, &block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
ret = this->update_head_block(block);
if (ailego_unlikely(ret != 0)) {
return ret;
}
ret = this->add_to_block(block, id, query, size);
if (ailego_unlikely(ret != 0)) {
return ret;
}
}
} else {
VectorLocation vector_loc = withid_key_info_map_[id];
auto segment = this->get_segment(vector_loc.segment_id);
size_t vector_off = vector_loc.offset;
if (segment->write(vector_off, query, size) != size) {
LOG_ERROR("Failed to write vector, off=%zu size=%u", vector_off, size);
return IndexError_WriteData;
}
size_t key_off = withid_key_map_[id];
uint64_t key = id;
if (segment->write(key_off, &key, sizeof(key)) != sizeof(key)) {
LOG_ERROR("Failed to write key, off=%zu", key_off);
return IndexError_WriteData;
}
key_info_map_lock_->lock();
key_info_map_[key] = vector_loc;
key_info_map_lock_->unlock();
}
(*stats_.mutable_added_count())++;
stats_.set_revision_id(meta_.revision_id + 1);
return 0;
}
} // namespace core
} // namespace zvec
@@ -0,0 +1,407 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <unordered_map>
#include <ailego/parallel/lock.h>
#include <ailego/utility/memory_helper.h>
#include <ailego/utility/string_helper.h>
#include <framework/index_context.h>
#include <framework/index_framework.h>
#include <framework/index_streamer.h>
#include "flat_index_format.h"
#include "flat_utility.h"
namespace zvec {
namespace core {
/*! Flat Streamer Entity
*/
class FlatStreamerEntity {
public:
typedef std::shared_ptr<FlatStreamerEntity> Pointer;
//! Constructor
explicit FlatStreamerEntity(IndexStreamer::Stats &stats);
//! Destructor
virtual ~FlatStreamerEntity(void) = default;
//! Open the entity with storage
int open(IndexStorage::Pointer storage, const IndexMeta &mt);
//! Close the entity
int close(void);
//! Flush Linear Meta information to storage
int flush_linear_meta(void);
//! Flush linear index to storage
int flush(uint64_t checkpoint);
//! Add vector to linear index
int add(uint64_t key, const void *vec, size_t size);
//! Search in linear list with filter
int search(const void *query, const IndexFilter &filter, uint32_t *scan_count,
IndexDocumentHeap *heap, IndexContext::Stats *context_stats) const;
//! Search in a block
void search_block(const void *query, const BlockLocation &bl,
const BlockHeader *hd, float norm_val,
IndexDocumentHeap *heap) const;
//! Search in a block with filter
void search_block(const void *query, const BlockLocation &bl,
const BlockHeader *hd, float norm_val,
const IndexFilter &filter, const DeletionMap *deletion_map,
IndexDocumentHeap *heap,
IndexContext::Stats *context_stats) const;
//! Flat Search with filter
int search_bf(const void *query, const IndexFilter &filter,
IndexDocumentHeap *heap,
IndexContext::Stats *context_stats) const;
//! Clone the entity
virtual FlatStreamerEntity::Pointer clone(void) const;
//! Retrieve the total vectors in the index
size_t vector_count(void) const {
return meta_.header.total_vector_count;
}
//! Retrieve the linear list count
size_t linear_list_count(void) const {
return meta_.header.linear_list_count;
}
//! Retrieve block size of the linear vector
size_t linear_block_size(void) const {
return meta_.header.block_size;
}
//! Retrieve the vectors count in one block
size_t block_vector_count(void) const {
// assert(meta_.header.block_vector_count == 32);
return meta_.header.block_vector_count;
}
//! Retrieve IndexMeta of the linear index
const IndexMeta &meta(void) const {
return index_meta_;
}
//! Retrieve mutable IndexMeta of the linear index
IndexMeta *mutable_meta(void) {
return &index_meta_;
}
//! Retrieve vector by local id
const void *get_vector_by_key(uint64_t key) const;
int get_vector_by_key(const uint64_t key,
IndexStorage::MemoryBlock &block) const;
//! Create a new iterator
IndexProvider::Iterator::Pointer creater_iterator(void) const;
//! Set params
void set_block_vector_count(uint32_t count) {
meta_.header.block_vector_count = count;
}
void set_use_key_info_map(bool use_id_map) {
use_key_info_map_ = use_id_map;
LOG_DEBUG("use_key_info_map_: %d", (int)use_key_info_map_);
}
//! Set params
void set_segment_size(uint32_t size) {
meta_.segment_size = size;
}
//! Set params
void set_linear_list_count(uint32_t count) {
meta_.header.linear_list_count = count;
}
//! Set params
void enable_filter_same_key(bool enabled) {
filter_same_key_ = enabled;
}
inline uint64_t key(uint32_t id) const {
if (id < id_key_vector_.size()) {
return id_key_vector_[id];
} else {
return kInvalidKey;
}
}
inline void row_major_distance(const void *query, const void *feature,
size_t fnum, float *out) const {
const uint8_t *cur_feature = reinterpret_cast<const uint8_t *>(feature);
for (size_t f = 0; f < fnum; ++f) {
row_distance_(query, cur_feature, index_meta_.dimension(), out + f);
cur_feature += index_meta_.element_size();
}
}
int add_vector_with_id(const uint32_t id, const void *query,
const uint32_t element_size);
private:
//! Disable them
FlatStreamerEntity(const FlatStreamerEntity &) = delete;
FlatStreamerEntity &operator=(const FlatStreamerEntity &) = delete;
/*! Iterator of all the linear list
*/
class Iterator : public IndexProvider::Iterator {
public:
//! Constructor
Iterator(const FlatStreamerEntity::Pointer &entity) : entity_(entity) {
this->read_next_block();
}
//! Retrieve pointer of data
const void *data(void) const override {
return reinterpret_cast<const char *>(data_) +
block_vector_index_ * entity_->index_meta_.element_size();
}
//! Test if the iterator is valid
bool is_valid(void) const override {
return is_valid_;
}
//! Retrieve primary key
uint64_t key(void) const override {
return keys_[block_vector_index_];
}
//! Next iterator
void next(void) override {
if (++block_vector_index_ == block_vector_count_) {
++block_index_;
this->read_next_block();
}
}
private:
//! Read next non-empty block
void read_next_block(void);
//! Members
std::string buffer_{};
const FlatStreamerEntity::Pointer entity_;
IndexStorage::MemoryBlock block_;
const void *data_{nullptr};
const uint64_t *keys_{nullptr};
uint32_t segment_id_{1u}; // The first segment is header info
uint32_t block_index_{0u};
uint32_t block_vector_index_{0u};
uint32_t block_vector_count_{0u};
bool is_valid_{true};
};
//! Retrive storage segment by index
const IndexStorage::Segment::Pointer get_segment(size_t index) const {
for (size_t i = segments_.size(); i <= index; ++i) {
auto segment_id =
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
auto segment = storage_->get(segment_id);
if (!segment) {
LOG_ERROR("Failed to get segment %s", segment_id.c_str());
return IndexStorage::Segment::Pointer();
}
segments_.emplace_back(std::move(segment));
}
return segments_[index];
}
//! Rejust the segment size as to aligned by page size
void AdjustSegmentSize(StreamerLinearMeta *mt) {
if (mt->segment_size < mt->header.block_size) {
mt->segment_size = mt->header.block_size;
}
mt->segment_size = ailego_align(
mt->segment_size / mt->header.block_size * mt->header.block_size,
ailego::MemoryHelper::PageSize());
}
//! Init with an empty storage
int init_storage(IndexStorage::Pointer storage);
//! Load linear meta information from storage
int load_linear_meta(IndexStorage::Pointer storage);
//! Load keys to keys map
int load_segment_keys_to_map(BlockLocation block);
//! Load keys to keys map
int load_segment_keys_to_vector(void);
//! Load index from storage
int load_storage(IndexStorage::Pointer storage);
//! Check whether the block is empty
bool is_valid_block(const BlockLocation &block) const {
return block.segment_id != 0;
}
//! Update header block of an linear list
int update_head_block(const BlockLocation &block) {
ailego_assert_with(segments_.size() != 0, "Invalid Segments");
auto &hd_segment = segments_[0];
if (hd_segment->write(0, &block, sizeof(block)) != sizeof(block)) {
LOG_ERROR("Failed to write head block location");
return IndexError_WriteData;
}
return 0;
}
//! Alloc a new segment
int alloc_segment(void);
//! Alloc a new block
int alloc_block(const BlockLocation &next, BlockLocation *block);
//! Add a record to a block
int add_to_block(const BlockLocation &block, uint64_t key, const void *data,
size_t size);
private:
size_t get_block_offset(uint32_t block_index) const {
return block_index * linear_block_size();
}
size_t get_block_header_offset(uint32_t block_index) const {
return get_block_offset(block_index) + linear_block_size() -
sizeof(BlockHeader);
}
size_t get_block_deletion_map_offset(uint32_t block_index) const {
return get_block_header_offset(block_index) - sizeof(DeletionMap);
}
size_t get_block_key_offset(uint32_t block_index,
uint32_t vector_index) const {
return get_block_offset(block_index) +
block_vector_count() * index_meta_.element_size() +
sizeof(uint64_t) * vector_index;
}
size_t get_block_vector_offset(uint32_t block_index,
uint32_t vector_index) const {
return this->get_block_offset(block_index) +
vector_index * index_meta_.element_size();
}
//! Get header block of an linear list
int get_head_block(IndexStorage::MemoryBlock &header_block) const {
ailego_assert_with(segments_.size() != 0, "Invalid Segments");
auto &hd_segment = segments_[0];
if (hd_segment->read(0, header_block, sizeof(BlockLocation)) !=
sizeof(BlockLocation)) {
LOG_ERROR("Failed to read head block location");
return -1;
}
return 0;
}
//! Get BlockHeader of the block
int get_block_header(const BlockLocation &block,
IndexStorage::MemoryBlock &header_block) const {
// The header is located in the end of a block to align features
auto &segment = this->get_segment(block.segment_id);
ailego_assert_with(segment != nullptr, "Index Overflow");
size_t off = this->get_block_header_offset(block.block_index);
if (segment->read(off, header_block, sizeof(BlockHeader)) !=
sizeof(BlockHeader)) {
LOG_ERROR("Failed to read block header, off=%zu", off);
return -1;
}
return 0;
}
int get_block_deletion_map(
const BlockLocation &block,
IndexStorage::MemoryBlock &deletion_map_block) const {
auto &segment = this->get_segment(block.segment_id);
ailego_assert_with(segment != nullptr, "Index Overflow");
size_t off = this->get_block_deletion_map_offset(block.block_index);
if (segment->read(off, deletion_map_block, sizeof(DeletionMap)) !=
sizeof(DeletionMap)) {
LOG_ERROR("Failed to read deletion map, off=%zu", off);
return -1;
}
return 0;
}
int get_block_keys(const BlockLocation &block,
IndexStorage::MemoryBlock &keys_block) const {
auto &segment = this->get_segment(block.segment_id);
ailego_assert_with(segment != nullptr, "Index Overflow");
size_t off = this->get_block_key_offset(block.block_index, 0);
if (segment->read(off, keys_block,
block_vector_count() * sizeof(uint64_t)) !=
block_vector_count() * sizeof(uint64_t)) {
LOG_ERROR("Failed to read block header, off=%zu", off);
return -1;
}
return 0;
}
int get_block_vectors(const BlockLocation &block,
IndexStorage::MemoryBlock &vector_block) const {
auto &segment = this->get_segment(block.segment_id);
ailego_assert_with(segment != nullptr, "Index Overflow");
size_t off = this->get_block_vector_offset(block.block_index, 0);
if (segment->read(off, vector_block,
block_vector_count() * index_meta_.element_size()) !=
block_vector_count() * index_meta_.element_size()) {
LOG_ERROR("Failed to read block header, off=%zu", off);
return -1;
}
return 0;
}
private:
//! Constants
static constexpr size_t kMaxSegmentId = std::numeric_limits<uint16_t>::max();
static constexpr size_t kMaxBlockId = std::numeric_limits<uint16_t>::max();
//! Members
std::mutex mutex_{};
IndexMeta index_meta_{};
IndexStorage::Pointer storage_{};
IndexMetric::MatrixDistance row_distance_{}, column_distance_{};
mutable std::vector<IndexStorage::Segment::Pointer> segments_{};
StreamerLinearMeta meta_{};
IndexStreamer::Stats &stats_;
mutable std::shared_ptr<ailego::SharedMutex> key_info_map_lock_{};
std::unordered_map<uint64_t, VectorLocation> key_info_map_{};
std::vector<VectorLocation> withid_key_info_map_{};
std::vector<uint32_t> withid_key_map_{};
std::vector<uint64_t> id_key_vector_{};
bool filter_same_key_{false};
bool use_key_info_map_{true};
uint32_t vec_unit_size_{0};
uint32_t vec_cols_{0};
mutable std::string vec_buf_{};
};
} // namespace core
} // namespace zvec
@@ -0,0 +1,100 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "flat_distance_matrix.h"
#include "flat_searcher.h"
#include "flat_streamer.h"
#include "flat_utility.h"
namespace zvec {
namespace core {
/*! Brute Force Streamer Provider
*/
template <size_t BATCH_SIZE>
class FlatStreamerProvider : public IndexProvider {
public:
//! Constructor
FlatStreamerProvider(const FlatStreamer<BATCH_SIZE> *owner) {
feature_size_ = owner->meta().element_size();
total_vector_count_ = owner->entity().vector_count();
owner_ = owner;
block_buffer_.resize(BATCH_SIZE * feature_size_);
}
//! Create a new iterator
IndexProvider::Iterator::Pointer create_iterator(void) override {
return owner_->entity().creater_iterator();
}
//! Retrieve count of vectors
size_t count(void) const override {
return total_vector_count_;
}
//! Retrieve dimension of vector
size_t dimension(void) const override {
return owner_->meta().dimension();
}
//! Retrieve type of vector
IndexMeta::DataType data_type(void) const override {
return owner_->meta().data_type();
}
//! Retrieve vector size in bytes
size_t element_size(void) const override {
return owner_->meta().element_size();
}
//! Retrieve a vector using a primary key
const void *get_vector(uint64_t key) const override {
return this->get_vector_by_key(key);
}
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return this->get_vector_by_key(key, block);
}
//! Retrieve the owner class
const std::string &owner_class(void) const override {
return owner_->name();
}
protected:
//! Retrieve a vector via primary key
const void *get_vector_by_key(uint64_t key) const {
return owner_->get_vector_by_key(key);
}
int get_vector_by_key(const uint64_t key,
IndexStorage::MemoryBlock &block) const {
return owner_->get_vector_by_key(key, block);
}
private:
//! Members
const FlatStreamer<BATCH_SIZE> *owner_{nullptr};
IndexStorage::Segment::Pointer features_segment_{};
uint32_t feature_size_{0};
uint32_t total_vector_count_{0};
mutable std::vector<uint8_t> block_buffer_{};
};
} // namespace core
} // namespace zvec
+201
View File
@@ -0,0 +1,201 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <mutex>
#include <ailego/utility/matrix_helper.h>
#include <ailego/utility/time_helper.h>
#include "framework/index_error.h"
#include "framework/index_factory.h"
#include "framework/index_meta.h"
#include "framework/index_metric.h"
namespace zvec {
namespace core {
//! The default size of reading a block
static constexpr uint32_t FLAT_READ_BLOCK_SIZE = 4 * 1024 * 1024;
static const std::string FLAT_LINEAR_META_SEG_ID = "linear.linear_meta";
static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID =
"linear.linear_list_head";
// reducer params
static const std::string PARAM_FLAT_REDUCER_WORKING_PATH(
"proxima.linear.reducer.working_path");
static const std::string PARAM_FLAT_REDUCER_NUM_OF_ADD_THREADS(
"proxima.linear.reducer.num_of_add_threads");
static const std::string PARAM_FLAT_REDUCER_INDEX_NAME(
"proxima.linear.reducer.index_name");
// index params
static const std::string PARAM_FLAT_SEGMENT_KEYS("linear.keys");
static const std::string PARAM_FLAT_SEGMENT_FEATURES("linear.features");
static const std::string PARAM_FLAT_SEGMENT_MAPPING("linear.mapping");
static const std::string FLAT_BUILDER_COLUMN_MAJOR_ORDER(
"proxima.linear.builder.column_major_order");
static const std::string FLAT_SEARCHER_BATCH_SIZE(
"proxima.linear.searcher.batch_size");
static const std::string FLAT_SEARCHER_READ_BLOCK_SIZE(
"proxima.linear.searcher.read_block_size");
static const std::string FLAT_SEARCHER_USE_ID_MAP(
"proxima.linear.searcher.use_id_map");
//! Determines if a number is equal to two to the power of n.
template <size_t K>
struct IsEqualPowerofTwo
: std::integral_constant<bool, K != 0 && (K ^ (K - 1)) == (K | (K - 1))> {};
//! Transpose a block
template <size_t M>
static inline void ReverseTranspose(size_t align_size, const void *src,
size_t dim, void *dst) {
switch (align_size) {
case 2:
ailego::MatrixHelper::ReverseTranspose<uint16_t, M>(src, dim, dst);
break;
case 4:
ailego::MatrixHelper::ReverseTranspose<uint32_t, M>(src, dim, dst);
break;
case 8:
ailego::MatrixHelper::ReverseTranspose<uint64_t, M>(src, dim, dst);
break;
}
}
static inline void ReverseTranspose(size_t align_size, const void *src,
size_t m, size_t dim, void *dst) {
switch (align_size) {
case 2:
ailego::MatrixHelper::ReverseTranspose<uint16_t>(src, m, dim, dst);
break;
case 4:
ailego::MatrixHelper::ReverseTranspose<uint32_t>(src, m, dim, dst);
break;
case 8:
ailego::MatrixHelper::ReverseTranspose<uint64_t>(src, m, dim, dst);
break;
}
}
template <typename T>
static inline void TransposeOne(const void *src, size_t M, size_t N,
void *dst) {
for (size_t i = 0; i < N; ++i) {
reinterpret_cast<T *>(dst)[i] = reinterpret_cast<const T *>(src)[i * M];
}
}
static inline void Transpose(size_t align_size, const void *src, size_t m,
size_t dim, void *dst) {
switch (align_size) {
case 2:
ailego::MatrixHelper::Transpose<uint16_t>(src, m, dim, dst);
break;
case 4:
ailego::MatrixHelper::Transpose<uint32_t>(src, m, dim, dst);
break;
case 8:
ailego::MatrixHelper::Transpose<uint64_t>(src, m, dim, dst);
break;
}
}
//! Transpose queries
template <size_t K>
void TransposeQueries(const void *query, const IndexQueryMeta &qmeta,
size_t query_count, std::string *out) {
if (K <= 1) {
ailego_assert(query_count == 1);
(void)query_count;
out->append(reinterpret_cast<const char *>(query) + out->size(),
qmeta.element_size());
} else {
ailego_assert_with(IsEqualPowerofTwo<K>::value,
"K must be equal to two to the power of n.");
size_t query_batch_count = query_count / K;
size_t query_offset = out->size();
out->resize(query_offset + query_batch_count * K * qmeta.element_size());
switch (IndexMeta::AlignSizeof(qmeta.data_type())) {
case 2:
for (size_t i = 0; i != query_batch_count; ++i) {
ailego::MatrixHelper::Transpose<uint16_t, K>(
(const char *)query + query_offset,
qmeta.element_size() / sizeof(uint16_t), &((*out)[query_offset]));
query_offset += qmeta.element_size() * K;
}
break;
case 4:
for (size_t i = 0; i != query_batch_count; ++i) {
ailego::MatrixHelper::Transpose<uint32_t, K>(
(const char *)query + query_offset,
qmeta.element_size() / sizeof(uint32_t), &((*out)[query_offset]));
query_offset += qmeta.element_size() * K;
}
break;
case 8:
for (size_t i = 0; i != query_batch_count; ++i) {
ailego::MatrixHelper::Transpose<uint64_t, K>(
(const char *)query + query_offset,
qmeta.element_size() / sizeof(uint64_t), &((*out)[query_offset]));
query_offset += qmeta.element_size() * K;
}
break;
default:
ailego_check_with(0, "BAD CASE");
}
size_t query_left_count = query_count % K;
if (query_left_count != 0) {
TransposeQueries<(K >> 1)>(query, qmeta, query_left_count, out);
}
}
}
//! Create and initialize measure
static inline int InitializeMetric(const IndexMeta &mt,
IndexMetric::Pointer *out) {
IndexMetric::Pointer measure = IndexFactory::CreateMetric(mt.metric_name());
if (!measure) {
return IndexError_NoExist;
}
int error_code = measure->init(mt, mt.metric_params());
if (error_code != 0) {
return error_code;
}
*out = measure;
return 0;
}
//! Verify measure
static inline bool VerifyMetric(const IndexMeta &meta) {
IndexMetric::Pointer measure = IndexFactory::CreateMetric(meta.metric_name());
if (!measure) {
return false;
}
int error_code = measure->init(meta, meta.metric_params());
if (error_code != 0) {
return false;
}
return true;
}
} // namespace core
} // namespace zvec