refactor: apply modernize-use-override across codebase (#442)

- Remove redundant `virtual` keyword from methods already marked `override`
- Replace `virtual ~Derived()` with `~Derived() override` for derived class destructors
- Add missing `override` on methods that override base class virtuals
This commit is contained in:
egolearner
2026-06-02 13:49:39 +08:00
committed by GitHub
parent df447a54e2
commit 3d6906ca20
56 changed files with 540 additions and 576 deletions
+1 -1
View File
@@ -27,7 +27,7 @@ template <size_t BATCH_SIZE>
class FlatBuilder : public IndexBuilder {
public:
//! Destructor
virtual ~FlatBuilder(void) {}
~FlatBuilder(void) override {}
//! Initialize the builder
int init(const IndexMeta &meta, const ailego::Params &params) override;
+1 -1
View File
@@ -27,7 +27,7 @@ template <size_t BATCH_SIZE>
class FlatSearcher : public IndexSearcher {
public:
//! Destructor
virtual ~FlatSearcher(void) = default;
~FlatSearcher(void) override = default;
//! Initialize Searcher
int init(const ailego::Params &index_params) override {
@@ -35,7 +35,7 @@ class FlatSearcherContext : public IndexSearcher::Context {
}
//! Destructor
virtual ~FlatSearcherContext(void) {}
~FlatSearcherContext(void) override {}
//! Set topk of search result
void set_topk(uint32_t topk) override {
@@ -58,13 +58,12 @@ class FlatSearcherContext : public IndexSearcher::Context {
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(void) const override {
const IndexGroupDocumentList &group_result(void) const override {
return group_results_[0];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(
size_t idx) const override {
const IndexGroupDocumentList &group_result(size_t idx) const override {
return group_results_[idx];
}
+4 -4
View File
@@ -30,7 +30,7 @@ class FlatStreamer : public IndexStreamer {
using ContextPointer = IndexStreamer::Context::UPointer;
FlatStreamer(void);
virtual ~FlatStreamer(void);
~FlatStreamer(void) override;
FlatStreamer(const FlatStreamer &streamer) = delete;
FlatStreamer &operator=(const FlatStreamer &streamer) = delete;
@@ -126,12 +126,12 @@ class FlatStreamer : public IndexStreamer {
return entity_;
}
virtual const void *get_vector(uint64_t key) const override {
const void *get_vector(uint64_t key) const override {
return this->get_vector_by_key(key);
}
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return this->get_vector_by_key(key, block);
}
@@ -30,7 +30,7 @@ class FlatStreamerContext : public IndexStreamer::Context {
}
//! Destructor
virtual ~FlatStreamerContext(void) = default;
~FlatStreamerContext(void) override = default;
//! Set topk of search result
void set_topk(uint32_t topk) override {
@@ -59,12 +59,11 @@ class FlatStreamerContext : public IndexStreamer::Context {
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(void) const override {
const IndexGroupDocumentList &group_result(void) const override {
return group_results_[0];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(
size_t idx) const override {
const IndexGroupDocumentList &group_result(size_t idx) const override {
return group_results_[idx];
}
@@ -45,7 +45,7 @@ class FlatSparseContext : public IndexContext {
: searcher_owner_(searcher_ptr), context_type_(kSearcherContext) {}
//! Destructor
virtual ~FlatSparseContext(void) = default;
~FlatSparseContext(void) override = default;
//! Set topk of search result
void set_topk(uint32_t topk) override {
@@ -100,33 +100,33 @@ class FlatSparseIndexProvider : public IndexSparseProvider {
}
//! Retrieve sparse count
virtual uint32_t sparse_count() const override {
uint32_t sparse_count() const override {
return sparse_count_;
}
//! Retrieve sparse indices
virtual const uint32_t *sparse_indices() const override {
const uint32_t *sparse_indices() const override {
return reinterpret_cast<const uint32_t *>(sparse_indices_buffer_.data());
}
//! Retrieve sparse data
virtual const void *sparse_data() const override {
const void *sparse_data() const override {
return reinterpret_cast<const void *>(sparse_data_buffer_.data());
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
bool is_valid(void) const override {
return cur_id_ < entity_->doc_cnt() && valid_;
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
uint64_t key(void) const override {
// std::cout << "iter key=" << cur_id_ << std::endl;
return entity_->get_key(cur_id_);
}
//! Next iterator
virtual void next(void) override {
void next(void) override {
cur_id_ = get_next_valid_id(cur_id_ + 1);
if (cur_id_ < entity_->doc_cnt()) {
@@ -28,7 +28,7 @@ class FlatSparseSearcher : public IndexSearcher {
public:
FlatSparseSearcher(void);
virtual ~FlatSparseSearcher(void);
~FlatSparseSearcher(void) override;
FlatSparseSearcher(const FlatSparseSearcher &) = delete;
FlatSparseSearcher &operator=(const FlatSparseSearcher &) = delete;
@@ -31,7 +31,7 @@ class FlatSparseStreamer : public IndexStreamer {
using ContextPointer = IndexStreamer::Context::Pointer;
FlatSparseStreamer(void);
virtual ~FlatSparseStreamer(void);
~FlatSparseStreamer(void) override;
FlatSparseStreamer(const FlatSparseStreamer &streamer) = delete;
FlatSparseStreamer &operator=(const FlatSparseStreamer &streamer) = delete;
+12 -13
View File
@@ -43,58 +43,57 @@ class HnswContext : public IndexContext {
const HnswEntity::Pointer &entity);
//! Destructor
virtual ~HnswContext();
~HnswContext() override;
public:
//! Set topk of search result
virtual void set_topk(uint32_t val) override {
void set_topk(uint32_t val) override {
topk_ = val;
topk_heap_.limit(std::max(val, ef_));
}
//! Retrieve search result
virtual const IndexDocumentList &result(void) const override {
const IndexDocumentList &result(void) const override {
return results_[0];
}
//! Retrieve search result
virtual const IndexDocumentList &result(size_t idx) const override {
const IndexDocumentList &result(size_t idx) const override {
return results_[idx];
}
//! Retrieve result object for output
virtual IndexDocumentList *mutable_result(size_t idx) override {
IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(void) const override {
const IndexGroupDocumentList &group_result(void) const override {
return group_results_[0];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(
size_t idx) const override {
const IndexGroupDocumentList &group_result(size_t idx) const override {
return group_results_[idx];
}
virtual uint32_t magic(void) const override {
uint32_t magic(void) const override {
return magic_;
}
//! Set mode of debug
virtual void set_debug_mode(bool enable) override {
void set_debug_mode(bool enable) override {
debug_mode_ = enable;
}
//! Retrieve mode of debug
virtual bool debug_mode(void) const override {
bool debug_mode(void) const override {
return this->debugging();
}
//! Retrieve string of debug
virtual std::string debug_string(void) const override {
std::string debug_string(void) const override {
char buf[4096];
size_t size = snprintf(
buf, sizeof(buf),
@@ -105,7 +104,7 @@ class HnswContext : public IndexContext {
}
//! Update the parameters of context
virtual int update(const ailego::Params &params) override;
int update(const ailego::Params &params) override;
public:
//! Init context
@@ -82,22 +82,22 @@ class HnswIndexProvider : public IndexProvider {
//! Retrieve pointer of data
//! NOTICE: the vec feature will be changed after iterating to next, so
//! the caller need to keep a copy of it before iterator to next vector
virtual const void *data(void) const override {
const void *data(void) const override {
return entity_->get_vector(cur_id_);
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
bool is_valid(void) const override {
return cur_id_ < entity_->doc_cnt();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
uint64_t key(void) const override {
return entity_->get_key(cur_id_);
}
//! Next iterator
virtual void next(void) override {
void next(void) override {
// cur_id_ += 1;
cur_id_ = get_next_valid_id(cur_id_ + 1);
}
+40 -43
View File
@@ -26,7 +26,7 @@ class HnswStreamer : public IndexStreamer {
using ContextPointer = IndexStreamer::Context::Pointer;
HnswStreamer(void);
virtual ~HnswStreamer(void);
~HnswStreamer(void) override;
HnswStreamer(const HnswStreamer &streamer) = delete;
HnswStreamer &operator=(const HnswStreamer &streamer) = delete;
@@ -44,102 +44,99 @@ class HnswStreamer : public IndexStreamer {
protected:
//! Initialize Streamer
virtual int init(const IndexMeta &imeta,
const ailego::Params &params) override;
int init(const IndexMeta &imeta, const ailego::Params &params) override;
//! Cleanup Streamer
virtual int cleanup(void) override;
int cleanup(void) override;
//! Create a context
virtual Context::Pointer create_context(void) const override;
Context::Pointer create_context(void) const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
IndexProvider::Pointer create_provider(void) const override;
//! Add a vector into index
virtual int add_impl(uint64_t pkey, const void *query,
int add_impl(uint64_t pkey, const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Add a vector with id into index
int add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Add a vector with id into index
virtual int add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Similarity search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
//! Fetch vector by key
virtual const void *get_vector(uint64_t key) const override {
const void *get_vector(uint64_t key) const override {
return entity_->get_vector_by_key(key);
}
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector_by_key(key, block);
}
//! Fetch vector by id
virtual const void *get_vector_by_id(uint32_t id) const override {
const void *get_vector_by_id(uint32_t id) const override {
return entity_->get_vector(id);
}
virtual int get_vector_by_id(
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
int get_vector_by_id(const uint32_t id,
IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector(id, block);
}
//! Open index from file path
virtual int open(IndexStorage::Pointer stg) override;
int open(IndexStorage::Pointer stg) override;
//! Close file
virtual int close(void) override;
int close(void) override;
//! flush file
virtual int flush(uint64_t checkpoint) override;
int flush(uint64_t checkpoint) override;
//! Dump index into storage
virtual int dump(const IndexDumper::Pointer &dumper) override;
int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
const Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
const IndexMeta &meta(void) const override {
return meta_;
}
virtual void print_debug_info() override;
void print_debug_info() override;
private:
inline int check_params(const void *query,
+21 -23
View File
@@ -43,56 +43,54 @@ class HnswStreamerEntity : public HnswEntity {
public:
//! Cleanup
//! return 0 on success, or errCode in failure
virtual int cleanup() override;
int cleanup() override;
//! Make a copy of streamer entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswEntity::Pointer clone() const override;
const HnswEntity::Pointer clone() const override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
key_t get_key(node_id_t id) const override;
//! Get vector feature data by key
virtual const void *get_vector(node_id_t id) const override;
const void *get_vector(node_id_t id) const override;
//! Get vectors feature data by local ids
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(
int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
//! Get the node id's neighbors on graph level
//! Note: the neighbors cannot be modified, using the following
//! method to get WritableNeighbors if want to
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
//! Add vector and key to hnsw entity, and local id will be saved in id
virtual int add_vector(level_t level, key_t key, const void *vec,
node_id_t *id) override;
int add_vector(level_t level, key_t key, const void *vec,
node_id_t *id) override;
//! Add vector and id to hnsw entity
virtual int add_vector_with_id(level_t level, node_id_t id,
const void *vec) override;
int add_vector_with_id(level_t level, node_id_t id, const void *vec) override;
virtual int update_neighbors(
int update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
//! Append neighbor_id to node id neighbors on level
//! Notice: the caller must be ensure the neighbors not full
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
//! Dump index by dumper
virtual int dump(const IndexDumper::Pointer &dumper) override;
int dump(const IndexDumper::Pointer &dumper) override;
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
void update_ep_and_level(node_id_t ep, level_t level) override;
//! Get the storage mode of this entity
virtual HnswStorageMode storage_mode() const {
@@ -112,13 +110,13 @@ class HnswStreamerEntity : public HnswEntity {
~HnswStreamerEntity();
//! Get vector feature data by key
virtual const void *get_vector_by_key(key_t key) const override {
const void *get_vector_by_key(key_t key) const override {
auto id = get_id(key);
return id == kInvalidNodeId ? nullptr : get_vector(id);
}
virtual int get_vector_by_key(
const key_t key, IndexStorage::MemoryBlock &block) const override {
int get_vector_by_key(const key_t key,
IndexStorage::MemoryBlock &block) const override {
auto id = get_id(key);
if (id != kInvalidNodeId) {
return get_vector(id, block);
@@ -44,58 +44,57 @@ class HnswRabitqContext : public IndexContext {
const HnswRabitqEntity::Pointer &entity);
//! Destructor
virtual ~HnswRabitqContext();
~HnswRabitqContext() override;
public:
//! Set topk of search result
virtual void set_topk(uint32_t val) override {
void set_topk(uint32_t val) override {
topk_ = val;
topk_heap_.limit(std::max(val, ef_));
}
//! Retrieve search result
virtual const IndexDocumentList &result(void) const override {
const IndexDocumentList &result(void) const override {
return results_[0];
}
//! Retrieve search result
virtual const IndexDocumentList &result(size_t idx) const override {
const IndexDocumentList &result(size_t idx) const override {
return results_[idx];
}
//! Retrieve result object for output
virtual IndexDocumentList *mutable_result(size_t idx) override {
IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(void) const override {
const IndexGroupDocumentList &group_result(void) const override {
return group_results_[0];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(
size_t idx) const override {
const IndexGroupDocumentList &group_result(size_t idx) const override {
return group_results_[idx];
}
virtual uint32_t magic(void) const override {
uint32_t magic(void) const override {
return magic_;
}
//! Set mode of debug
virtual void set_debug_mode(bool enable) override {
void set_debug_mode(bool enable) override {
debug_mode_ = enable;
}
//! Retrieve mode of debug
virtual bool debug_mode(void) const override {
bool debug_mode(void) const override {
return this->debugging();
}
//! Retrieve string of debug
virtual std::string debug_string(void) const override {
std::string debug_string(void) const override {
char buf[4096];
size_t size = snprintf(
buf, sizeof(buf),
@@ -106,7 +105,7 @@ class HnswRabitqContext : public IndexContext {
}
//! Update the parameters of context
virtual int update(const ailego::Params &params) override;
int update(const ailego::Params &params) override;
public:
//! Init context
@@ -83,22 +83,22 @@ class HnswRabitqIndexProvider : public IndexProvider {
//! Retrieve pointer of data
//! NOTICE: the vec feature will be changed after iterating to next, so
//! the caller need to keep a copy of it before iterator to next vector
virtual const void *data(void) const override {
const void *data(void) const override {
return entity_->get_vector(cur_id_);
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
bool is_valid(void) const override {
return cur_id_ < entity_->doc_cnt();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
uint64_t key(void) const override {
return entity_->get_key(cur_id_);
}
//! Next iterator
virtual void next(void) override {
void next(void) override {
// cur_id_ += 1;
cur_id_ = get_next_valid_id(cur_id_ + 1);
}
@@ -33,7 +33,7 @@ class HnswRabitqStreamer : public IndexStreamer {
HnswRabitqStreamer();
explicit HnswRabitqStreamer(IndexProvider::Pointer provider,
RabitqReformer::Pointer reformer = nullptr);
virtual ~HnswRabitqStreamer(void);
~HnswRabitqStreamer(void) override;
HnswRabitqStreamer(const HnswRabitqStreamer &streamer) = delete;
HnswRabitqStreamer &operator=(const HnswRabitqStreamer &streamer) = delete;
@@ -48,102 +48,99 @@ class HnswRabitqStreamer : public IndexStreamer {
protected:
//! Initialize Streamer
virtual int init(const IndexMeta &imeta,
const ailego::Params &params) override;
int init(const IndexMeta &imeta, const ailego::Params &params) override;
//! Cleanup Streamer
virtual int cleanup(void) override;
int cleanup(void) override;
//! Create a context
virtual Context::Pointer create_context(void) const override;
Context::Pointer create_context(void) const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
IndexProvider::Pointer create_provider(void) const override;
//! Add a vector into index
virtual int add_impl(uint64_t pkey, const void *query,
int add_impl(uint64_t pkey, const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Add a vector with id into index
int add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Add a vector with id into index
virtual int add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
//! Similarity search
int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta,
ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
//! Linear search by primary keys
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
//! Fetch vector by key
virtual const void *get_vector(uint64_t key) const override {
const void *get_vector(uint64_t key) const override {
return entity_.get_vector_by_key(key);
}
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_.get_vector_by_key(key, block);
}
//! Fetch vector by id
virtual const void *get_vector_by_id(uint32_t id) const override {
const void *get_vector_by_id(uint32_t id) const override {
return entity_.get_vector(id);
}
virtual int get_vector_by_id(
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
int get_vector_by_id(const uint32_t id,
IndexStorage::MemoryBlock &block) const override {
return entity_.get_vector(id, block);
}
//! Open index from file path
virtual int open(IndexStorage::Pointer stg) override;
int open(IndexStorage::Pointer stg) override;
//! Close file
virtual int close(void) override;
int close(void) override;
//! flush file
virtual int flush(uint64_t checkpoint) override;
int flush(uint64_t checkpoint) override;
//! Dump index into storage
virtual int dump(const IndexDumper::Pointer &dumper) override;
int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
const Stats &stats(void) const override {
return stats_;
}
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
const IndexMeta &meta(void) const override {
return meta_;
}
virtual void print_debug_info() override;
void print_debug_info() override;
private:
inline int check_params(const void *query,
@@ -34,57 +34,54 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
public:
//! Cleanup
//! return 0 on success, or errCode in failure
virtual int cleanup() override;
int cleanup() override;
//! Make a copy of streamer entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswRabitqEntity::Pointer clone() const override;
const HnswRabitqEntity::Pointer clone() const override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
key_t get_key(node_id_t id) const override;
//! Get vector feature data by key
virtual const void *get_vector(node_id_t id) const override;
const void *get_vector(node_id_t id) const override;
//! Get vectors feature data by local ids
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(
int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
//! Get the node id's neighbors on graph level
//! Note: the neighbors cannot be modified, using the following
//! method to get WritableNeighbors if want to
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
//! Add vector and key to hnsw entity, and local id will be saved in id
virtual int add_vector(level_t level, key_t key, const void *vec,
node_id_t *id) override;
int add_vector(level_t level, key_t key, const void *vec,
node_id_t *id) override;
//! Add vector and id to hnsw entity
virtual int add_vector_with_id(level_t level, node_id_t id,
const void *vec) override;
int add_vector_with_id(level_t level, node_id_t id, const void *vec) override;
virtual int update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors)
override;
int update_neighbors(level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, ResultRecord>>
&neighbors) override;
//! Append neighbor_id to node id neighbors on level
//! Notice: the caller must be ensure the neighbors not full
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
//! Dump index by dumper
virtual int dump(const IndexDumper::Pointer &dumper) override;
int dump(const IndexDumper::Pointer &dumper) override;
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
void update_ep_and_level(node_id_t ep, level_t level) override;
void set_use_key_info_map(bool use_id_map) {
use_key_info_map_ = use_id_map;
@@ -99,13 +96,13 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
~HnswRabitqStreamerEntity();
//! Get vector feature data by key
virtual const void *get_vector_by_key(key_t key) const override {
const void *get_vector_by_key(key_t key) const override {
auto id = get_id(key);
return id == kInvalidNodeId ? nullptr : get_vector(id);
}
virtual int get_vector_by_key(
const key_t key, IndexStorage::MemoryBlock &block) const override {
int get_vector_by_key(const key_t key,
IndexStorage::MemoryBlock &block) const override {
auto id = get_id(key);
if (id != kInvalidNodeId) {
return get_vector(id, block);
@@ -22,23 +22,23 @@ namespace core {
class HnswSparseBuilderEntity : public HnswSparseEntity {
public:
//! Add vector and key to hnsw entity, and local id will be saved in id
virtual int add_vector(level_t level, key_t key, const uint32_t sparse_count,
const uint32_t *sparse_indices, const void *sparse_vec,
node_id_t *id) override;
int add_vector(level_t level, key_t key, const uint32_t sparse_count,
const uint32_t *sparse_indices, const void *sparse_vec,
node_id_t *id) override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
key_t get_key(node_id_t id) const override;
//! Get vector feature data by key
virtual const void *get_vector_meta(node_id_t id) const override;
const void *get_vector_meta(node_id_t id) const override;
virtual int get_vector_meta(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
int get_vector_meta(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
//! Batch get vectors feature data by keys
virtual int get_vector_metas(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector_metas(
int get_vector_metas(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
int get_vector_metas(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &block_vecs) const override;
@@ -57,43 +57,41 @@ class HnswSparseBuilderEntity : public HnswSparseEntity {
}
//! Get the node id's neighbors on graph level
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
//! Replace node id in level's neighbors
virtual int update_neighbors(
int update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
//! add a neighbor to id in graph level
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
//! Get vector sparse feature data by chunk index and offset
virtual const void *get_sparse_data(uint64_t offset,
uint32_t len) const override;
const void *get_sparse_data(uint64_t offset, uint32_t len) const override;
//! Get sparse data from id
virtual const void *get_sparse_data(node_id_t id) const override;
const void *get_sparse_data(node_id_t id) const override;
virtual int get_sparse_data(uint64_t offset, uint32_t len,
IndexStorage::MemoryBlock &block) const override;
int get_sparse_data(uint64_t offset, uint32_t len,
IndexStorage::MemoryBlock &block) const override;
virtual int get_sparse_data(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
int get_sparse_data(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
//! Get sparse data from vector
virtual std::pair<const void *, uint32_t> get_sparse_data_from_vector(
std::pair<const void *, uint32_t> get_sparse_data_from_vector(
const void *vec) const override;
virtual int get_sparse_data_from_vector(const void *vec,
IndexStorage::MemoryBlock &block,
int &sparse_length) const override;
int get_sparse_data_from_vector(const void *vec,
IndexStorage::MemoryBlock &block,
int &sparse_length) const override;
//! Dump the hnsw graph to dumper
virtual int dump(const IndexDumper::Pointer &dumper) override;
int dump(const IndexDumper::Pointer &dumper) override;
//! Cleanup the entity
virtual int cleanup(void) override;
int cleanup(void) override;
public:
//! Constructor
@@ -38,58 +38,57 @@ class HnswSparseContext : public IndexContext {
const HnswSparseEntity::Pointer &entity);
//! Destructor
virtual ~HnswSparseContext();
~HnswSparseContext() override;
public:
//! Set topk of search result
virtual void set_topk(uint32_t val) override {
void set_topk(uint32_t val) override {
topk_ = val;
topk_heap_.limit(std::max(val, ef_));
}
//! Retrieve search result
virtual const IndexDocumentList &result(void) const override {
const IndexDocumentList &result(void) const override {
return results_[0];
}
//! Retrieve search result
virtual const IndexDocumentList &result(size_t idx) const override {
const IndexDocumentList &result(size_t idx) const override {
return results_[idx];
}
//! Retrieve result object for output
virtual IndexDocumentList *mutable_result(size_t idx) override {
IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(void) const override {
const IndexGroupDocumentList &group_result(void) const override {
return group_results_[0];
}
//! Retrieve search group result with index
virtual const IndexGroupDocumentList &group_result(
size_t idx) const override {
const IndexGroupDocumentList &group_result(size_t idx) const override {
return group_results_[idx];
}
virtual uint32_t magic(void) const override {
uint32_t magic(void) const override {
return magic_;
}
//! Set mode of debug
virtual void set_debug_mode(bool enable) override {
void set_debug_mode(bool enable) override {
debug_mode_ = enable;
}
//! Retrieve mode of debug
virtual bool debug_mode(void) const override {
bool debug_mode(void) const override {
return this->debugging();
}
//! Retrieve string of debug
virtual std::string debug_string(void) const override {
std::string debug_string(void) const override {
char buf[4096];
size_t size = snprintf(
buf, sizeof(buf),
@@ -100,7 +99,7 @@ class HnswSparseContext : public IndexContext {
}
//! Update the parameters of context
virtual int update(const ailego::Params &params) override;
int update(const ailego::Params &params) override;
public:
//! Init context
@@ -313,7 +312,7 @@ class HnswSparseContext : public IndexContext {
force_padding_topk_ = v;
}
virtual void set_bruteforce_threshold(uint32_t v) override {
void set_bruteforce_threshold(uint32_t v) override {
bruteforce_threshold_ = v;
}
@@ -321,11 +320,11 @@ class HnswSparseContext : public IndexContext {
return bruteforce_threshold_;
}
virtual void set_fetch_vector(bool v) override {
void set_fetch_vector(bool v) override {
fetch_vector_ = v;
}
virtual bool fetch_vector() const override {
bool fetch_vector() const override {
return fetch_vector_;
}
@@ -81,32 +81,32 @@ class HnswSparseIndexProvider : public IndexSparseProvider {
}
//! Retrieve sparse count
virtual uint32_t sparse_count() const override {
uint32_t sparse_count() const override {
return sparse_count_;
}
//! Retrieve sparse indices
virtual const uint32_t *sparse_indices() const override {
const uint32_t *sparse_indices() const override {
return reinterpret_cast<const uint32_t *>(sparse_indices_buffer_.data());
}
//! Retrieve sparse data
virtual const void *sparse_data() const override {
const void *sparse_data() const override {
return reinterpret_cast<const void *>(sparse_data_buffer_.data());
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
bool is_valid(void) const override {
return cur_id_ < entity_->doc_cnt() && valid_;
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
uint64_t key(void) const override {
return entity_->get_key(cur_id_);
}
//! Next iterator
virtual void next(void) override {
void next(void) override {
cur_id_ = get_next_valid_id(cur_id_ + 1);
if (cur_id_ < entity_->doc_cnt()) {
@@ -26,7 +26,7 @@ class HnswSparseSearcher : public IndexSearcher {
public:
HnswSparseSearcher(void);
virtual ~HnswSparseSearcher(void);
~HnswSparseSearcher(void) override;
HnswSparseSearcher(const HnswSparseSearcher &) = delete;
HnswSparseSearcher &operator=(const HnswSparseSearcher &) = delete;
@@ -46,64 +46,61 @@ class HnswSparseSearcherEntity : public HnswSparseEntity {
//! Make a copy of searcher entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswSparseEntity::Pointer clone() const override;
const HnswSparseEntity::Pointer clone() const override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
key_t get_key(node_id_t id) const override;
//! Get vector local id by key
node_id_t get_id(key_t key) const;
//! Get sparse vector feature data by key
virtual int get_sparse_vector_by_key(
int get_sparse_vector_by_key(
key_t key, uint32_t *sparse_count, std::string *sparse_indices_buffer,
std::string *sparse_values_buffer) const override;
//! Get vector feature data by id
virtual const void *get_vector_meta(node_id_t id) const override;
const void *get_vector_meta(node_id_t id) const override;
virtual int get_vector_meta(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
int get_vector_meta(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
//! Get vector feature data by id
virtual int get_vector_metas(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
int get_vector_metas(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector_metas(
int get_vector_metas(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &block_vecs) const override;
//! Get vector sparse feature data by chunk index and offset
virtual const void *get_sparse_data(uint64_t offset,
uint32_t len) const override;
const void *get_sparse_data(uint64_t offset, uint32_t len) const override;
//! Get sparse data from id
virtual const void *get_sparse_data(node_id_t id) const override;
const void *get_sparse_data(node_id_t id) const override;
virtual int get_sparse_data(uint64_t offset, uint32_t len,
IndexStorage::MemoryBlock &block) const override;
int get_sparse_data(uint64_t offset, uint32_t len,
IndexStorage::MemoryBlock &block) const override;
virtual int get_sparse_data(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
int get_sparse_data(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
//! Get sparse data from vector
virtual std::pair<const void *, uint32_t> get_sparse_data_from_vector(
std::pair<const void *, uint32_t> get_sparse_data_from_vector(
const void *vec) const override;
virtual int get_sparse_data_from_vector(const void *vec,
IndexStorage::MemoryBlock &block,
int &sparse_length) const override;
int get_sparse_data_from_vector(const void *vec,
IndexStorage::MemoryBlock &block,
int &sparse_length) const override;
//! Get the node id's neighbors on graph level
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
virtual int load(const IndexStorage::Pointer &container,
bool check_crc) override;
int load(const IndexStorage::Pointer &container, bool check_crc) override;
int load_segments(bool check_crc);
virtual int cleanup(void) override;
int cleanup(void) override;
public:
bool is_loaded() const {
@@ -26,7 +26,7 @@ class HnswSparseStreamer : public IndexStreamer {
using ContextPointer = IndexStreamer::Context::Pointer;
HnswSparseStreamer(void);
virtual ~HnswSparseStreamer(void);
~HnswSparseStreamer(void) override;
HnswSparseStreamer(const HnswSparseStreamer &streamer) = delete;
HnswSparseStreamer &operator=(const HnswSparseStreamer &streamer) = delete;
@@ -32,75 +32,72 @@ class HnswSparseStreamerEntity : public HnswSparseEntity {
public:
//! Cleanup
//! return 0 on success, or errCode in failure
virtual int cleanup() override;
int cleanup() override;
//! Make a copy of streamer entity, to support thread-safe operation.
//! The segment in container cannot be read concurrenly
virtual const HnswSparseEntity::Pointer clone() const override;
const HnswSparseEntity::Pointer clone() const override;
//! Get primary key of the node id
virtual key_t get_key(node_id_t id) const override;
key_t get_key(node_id_t id) const override;
//! Get vector feature data by key
virtual const void *get_vector_meta(node_id_t id) const override;
const void *get_vector_meta(node_id_t id) const override;
virtual int get_vector_meta(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
int get_vector_meta(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
//! Get vectors feature data by local ids
virtual int get_vector_metas(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector_metas(
int get_vector_metas(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
int get_vector_metas(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &block_vecs) const override;
//! Get vector sparse feature data by chunk index and offset
virtual const void *get_sparse_data(uint64_t offset,
uint32_t len) const override;
const void *get_sparse_data(uint64_t offset, uint32_t len) const override;
//! Get sparse data from id
virtual const void *get_sparse_data(node_id_t id) const override;
const void *get_sparse_data(node_id_t id) const override;
virtual int get_sparse_data(uint64_t offset, uint32_t len,
IndexStorage::MemoryBlock &block) const override;
virtual int get_sparse_data(node_id_t id,
IndexStorage::MemoryBlock &block) const override;
int get_sparse_data(uint64_t offset, uint32_t len,
IndexStorage::MemoryBlock &block) const override;
int get_sparse_data(node_id_t id,
IndexStorage::MemoryBlock &block) const override;
//! Get sparse data from vector
virtual std::pair<const void *, uint32_t> get_sparse_data_from_vector(
std::pair<const void *, uint32_t> get_sparse_data_from_vector(
const void *vec) const override;
virtual int get_sparse_data_from_vector(const void *vec,
IndexStorage::MemoryBlock &block,
int &sparse_length) const override;
int get_sparse_data_from_vector(const void *vec,
IndexStorage::MemoryBlock &block,
int &sparse_length) const override;
//! Get sparse vector feature data by key
virtual int get_sparse_vector_by_key(
int get_sparse_vector_by_key(
key_t key, uint32_t *sparse_count, std::string *sparse_indices_buffer,
std::string *sparse_values_buffer) const override;
//! Get sparse vector feature data by id
virtual int get_sparse_vector_by_id(
node_id_t id, uint32_t *sparse_count, std::string *sparse_indices_buffer,
std::string *sparse_values_buffer) const override;
int get_sparse_vector_by_id(node_id_t id, uint32_t *sparse_count,
std::string *sparse_indices_buffer,
std::string *sparse_values_buffer) const override;
//! Get the node id's neighbors on graph level
//! Note: the neighbors cannot be modified, using the following
//! method to get WritableNeighbors if want to
virtual const Neighbors get_neighbors(level_t level,
node_id_t id) const override;
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
//! Add vector and key to hnsw entity, and local id will be saved in id
virtual int add_vector(level_t level, key_t key,
const std::string &sparse_vec_buffer,
uint32_t sparse_count, node_id_t *id) override;
int add_vector(level_t level, key_t key, const std::string &sparse_vec_buffer,
uint32_t sparse_count, node_id_t *id) override;
//! Add vector and id to hnsw entity
virtual int add_vector_with_id(level_t level, node_id_t id,
const std::string &sparse_vec,
uint32_t sparse_count) override;
int add_vector_with_id(level_t level, node_id_t id,
const std::string &sparse_vec,
uint32_t sparse_count) override;
virtual int update_neighbors(
int update_neighbors(
level_t level, node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
@@ -116,13 +113,13 @@ class HnswSparseStreamerEntity : public HnswSparseEntity {
//! Append neighbor_id to node id neighbors on level
//! Notice: the caller must be ensure the neighbors not full
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
void add_neighbor(level_t level, node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
//! Dump index by dumper
virtual int dump(const IndexDumper::Pointer &dumper) override;
int dump(const IndexDumper::Pointer &dumper) override;
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
void update_ep_and_level(node_id_t ep, level_t level) override;
public:
//! Constructor
+21 -22
View File
@@ -28,7 +28,7 @@ class IVFBuilder : public IndexBuilder {
IVFBuilder();
//! Destructor
~IVFBuilder();
~IVFBuilder() override;
//! Disable them
IVFBuilder(const IVFBuilder &) = delete;
@@ -36,28 +36,27 @@ class IVFBuilder : public IndexBuilder {
public:
//! Initialize the builder
virtual int init(const IndexMeta &meta,
const ailego::Params &params) override;
int init(const IndexMeta &meta, const ailego::Params &params) override;
//! Cleanup the builder
virtual int cleanup(void) override;
int cleanup(void) override;
//! Train the data
virtual int train(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
int train(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
//! Train the data
virtual int train(const IndexTrainer::Pointer &trainer) override;
int train(const IndexTrainer::Pointer &trainer) override;
//! Build the index
virtual int build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
int build(IndexThreads::Pointer threads,
IndexHolder::Pointer holder) override;
//! Dump index into file system
virtual int dump(const IndexDumper::Pointer &dumper) override;
int dump(const IndexDumper::Pointer &dumper) override;
//! Retrieve statistics
virtual const Stats &stats(void) const override {
const Stats &stats(void) const override {
return stats_;
}
@@ -84,25 +83,25 @@ class IVFBuilder : public IndexBuilder {
Iterator(RandomAccessIndexHolder *owner) : holder_(owner) {}
//! Destructor
virtual ~Iterator(void) {}
~Iterator(void) override {}
//! Retrieve pointer of data
virtual const void *data(void) const override {
const void *data(void) const override {
return holder_->element(id_);
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
bool is_valid(void) const override {
return id_ < holder_->count();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
uint64_t key(void) const override {
return holder_->key(id_);
}
//! Next iterator
virtual void next(void) override {
void next(void) override {
++id_;
}
@@ -117,32 +116,32 @@ class IVFBuilder : public IndexBuilder {
: features_(std::make_shared<CompactIndexFeatures>(meta)) {}
//! Retrieve count of elements in holder (-1 indicates unknown)
virtual size_t count(void) const override {
size_t count(void) const override {
return features_->count();
}
//! Retrieve dimension
virtual size_t dimension(void) const override {
size_t dimension(void) const override {
return features_->dimension();
}
//! Retrieve type information
virtual IndexMeta::DataType data_type(void) const override {
IndexMeta::DataType data_type(void) const override {
return features_->data_type();
}
//! Retrieve element size in bytes
virtual size_t element_size(void) const override {
size_t element_size(void) const override {
return features_->element_size();
}
//! Retrieve if it can multi-pass
virtual bool multipass(void) const override {
bool multipass(void) const override {
return true;
}
//! Create a new iterator
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
IndexHolder::Iterator::Pointer create_iterator(void) override {
return IndexHolder::Iterator::Pointer(
new RandomAccessIndexHolder::Iterator(this));
}
+11 -11
View File
@@ -114,25 +114,25 @@ class IVFCentroidIndex {
Iterator(std::vector<const void *> *features) : features_(features) {}
//! Destructor
virtual ~Iterator(void) {}
~Iterator(void) override {}
//! Retrieve pointer of data
virtual const void *data(void) const override {
const void *data(void) const override {
return (*features_)[id_];
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
bool is_valid(void) const override {
return id_ < features_->size();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
uint64_t key(void) const override {
return id_;
}
//! Next iterator
virtual void next(void) override {
void next(void) override {
++id_;
}
@@ -168,32 +168,32 @@ class IVFCentroidIndex {
}
//! Retrieve count of elements in holder (-1 indicates unknown)
virtual size_t count(void) const override {
size_t count(void) const override {
return features_.size();
}
//! Retrieve dimension
virtual size_t dimension(void) const override {
size_t dimension(void) const override {
return dimension_;
}
//! Retrieve type information
virtual IndexMeta::DataType data_type(void) const override {
IndexMeta::DataType data_type(void) const override {
return data_type_;
}
//! Retrieve element size in bytes
virtual size_t element_size(void) const override {
size_t element_size(void) const override {
return element_size_;
}
//! Retrieve if it can multi-pass
virtual bool multipass(void) const override {
bool multipass(void) const override {
return true;
}
//! Create a new iterator
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
IndexHolder::Iterator::Pointer create_iterator(void) override {
return IndexHolder::Iterator::Pointer(
new CentroidsIndexHolder::Iterator(&features_));
}
+15 -15
View File
@@ -35,37 +35,37 @@ class IVFIndexProvider : public IndexProvider {
public:
//! Create a new iterator
virtual Iterator::Pointer create_iterator(void) override {
Iterator::Pointer create_iterator(void) override {
return Iterator::Pointer(new (std::nothrow) SortedIterator(entity_));
}
//! Retrieve count of vectors
virtual size_t count(void) const override {
size_t count(void) const override {
return entity_->vector_count();
}
//! Retrieve dimension of vector
virtual size_t dimension(void) const override {
size_t dimension(void) const override {
return meta_.dimension();
}
//! Retrieve type of vector
virtual IndexMeta::DataType data_type(void) const override {
IndexMeta::DataType data_type(void) const override {
return meta_.data_type();
}
//! Retrieve vector size in bytes
virtual size_t element_size(void) const override {
size_t element_size(void) const override {
return meta_.element_size();
}
//! Retrieve a vector using a primary key
virtual const void *get_vector(uint64_t key) const override {
const void *get_vector(uint64_t key) const override {
return entity_->get_vector_by_key(key);
}
//! Retrieve the owner class
virtual const std::string &owner_class(void) const override {
const std::string &owner_class(void) const override {
return owner_class_;
}
@@ -88,22 +88,22 @@ class IVFIndexProvider : public IndexProvider {
//! Retrieve pointer of data
//! NOTICE: the vec feature will be changed after iterating to next, so
//! the caller need to keep a copy of it before iterator to next vector
virtual const void *data(void) const override {
const void *data(void) const override {
return entity_->get_vector(current_local_id());
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
bool is_valid(void) const override {
return pos_ < count_;
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
uint64_t key(void) const override {
return entity_->get_key(current_local_id());
}
//! Next iterator
virtual void next(void) override {
void next(void) override {
++pos_;
}
@@ -126,22 +126,22 @@ class IVFIndexProvider : public IndexProvider {
Iterator(const IVFEntity::Pointer &entity) : entity_(entity) {}
//! Retrieve pointer of data
virtual const void *data(void) const override {
const void *data(void) const override {
return entity_->get_vector(index_);
}
//! Test if the iterator is valid
virtual bool is_valid(void) const override {
bool is_valid(void) const override {
return index_ < entity_->vector_count();
}
//! Retrieve primary key
virtual uint64_t key(void) const override {
uint64_t key(void) const override {
return entity_->get_key(index_);
}
//! Next iterator
virtual void next(void) override {
void next(void) override {
++index_;
}
+18 -20
View File
@@ -26,52 +26,50 @@ namespace core {
class IVFSearcher : public IndexSearcher {
public:
//! Initialize Searcher
virtual int init(const ailego::Params &parameters) override;
int init(const ailego::Params &parameters) override;
//! Cleanup Searcher
virtual int cleanup(void) override;
int cleanup(void) override;
//! Load index from container
virtual int load(IndexStorage::Pointer container,
IndexMetric::Pointer metric) override;
int load(IndexStorage::Pointer container,
IndexMetric::Pointer metric) override;
//! Unload index
virtual int unload(void) override;
int unload(void) override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
//! Retrieve statistics
virtual const Stats &stats(void) const override;
const Stats &stats(void) const override;
//! Create a searcher context
virtual Context::Pointer create_context(void) const override;
Context::Pointer create_context(void) const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
IndexProvider::Pointer create_provider(void) const override;
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
const IndexMeta &meta(void) const override {
return meta_;
}
//! Retrieve params of index
virtual const ailego::Params &params(void) const override {
const ailego::Params &params(void) const override {
return params_;
}
@@ -30,25 +30,25 @@ class IVFSearcherContext : public IndexSearcher::Context {
public:
//! Set topk of search result
virtual void set_topk(uint32_t k) override {
void set_topk(uint32_t k) override {
topk_ = k;
result_heap_.limit(topk_);
result_heap_.set_threshold(this->threshold());
}
//! Retrieve search result
virtual const IndexDocumentList &result(void) const override {
const IndexDocumentList &result(void) const override {
return results_[0];
}
//! Retrieve search result with index
virtual const IndexDocumentList &result(size_t idx) const override {
const IndexDocumentList &result(size_t idx) const override {
ailego_assert_with(results_.size() > idx, "invalid index");
return results_[idx];
}
//! Retrieve mutable result with index
virtual IndexDocumentList *mutable_result(size_t idx) override {
IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
@@ -58,7 +58,7 @@ class IVFSearcherContext : public IndexSearcher::Context {
}
//! Update the parameters of context
virtual int update(const ailego::Params &params) override {
int update(const ailego::Params &params) override {
params.get(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD,
&bruteforce_threshold_);
params.get(PARAM_IVF_SEARCHER_SCAN_RATIO, &scan_ratio_);
@@ -79,7 +79,7 @@ class IVFSearcherContext : public IndexSearcher::Context {
}
//! Retrieve magic number
virtual uint32_t magic(void) const override {
uint32_t magic(void) const override {
return magic_;
}
+21 -23
View File
@@ -27,59 +27,57 @@ namespace core {
class IVFStreamer : public IndexStreamer {
public:
//! Initialize Searcher
virtual int init(const IndexMeta & /*meta*/,
const ailego::Params & /*params*/) override;
int init(const IndexMeta & /*meta*/,
const ailego::Params & /*params*/) override;
//! Cleanup Searcher
virtual int cleanup(void) override;
int cleanup(void) override;
//! Load index from container
virtual int open(IndexStorage::Pointer storage) override;
int open(IndexStorage::Pointer storage) override;
virtual int flush(uint64_t /*check_point*/) override {
int flush(uint64_t /*check_point*/) override {
return 0;
}
virtual int close(void) override {
int close(void) override {
return this->unload();
}
//! Unload index
virtual int unload(void) override;
int unload(void) override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity brute force search
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
//! Similarity search
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
//! Retrieve statistics
virtual const Stats &stats(void) const override;
const Stats &stats(void) const override;
//! Create a searcher context
virtual Context::Pointer create_context(void) const override;
Context::Pointer create_context(void) const override;
//! Create a new iterator
virtual IndexProvider::Pointer create_provider(void) const override;
IndexProvider::Pointer create_provider(void) const override;
//! Retrieve meta of index
virtual const IndexMeta &meta(void) const override {
const IndexMeta &meta(void) const override {
return meta_;
}
virtual int get_vector_by_id(
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
int get_vector_by_id(const uint32_t id,
IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector_by_key(id, block);
}
+10 -10
View File
@@ -38,44 +38,44 @@ class VamanaContext : public IndexContext {
VamanaContext(const IndexMetric::Pointer &metric,
const VamanaEntity::Pointer &entity);
virtual ~VamanaContext();
~VamanaContext() override;
virtual void set_topk(uint32_t val) override {
void set_topk(uint32_t val) override {
topk_ = val;
topk_heap_.limit(std::max(val, ef_));
}
virtual const IndexDocumentList &result(void) const override {
const IndexDocumentList &result(void) const override {
return results_[0];
}
virtual const IndexDocumentList &result(size_t idx) const override {
const IndexDocumentList &result(size_t idx) const override {
return results_[idx];
}
virtual IndexDocumentList *mutable_result(size_t idx) override {
IndexDocumentList *mutable_result(size_t idx) override {
ailego_assert_with(idx < results_.size(), "invalid idx");
return &results_[idx];
}
virtual uint32_t magic(void) const override {
uint32_t magic(void) const override {
return magic_;
}
virtual void set_debug_mode(bool enable) override {
void set_debug_mode(bool enable) override {
debug_mode_ = enable;
}
virtual bool debug_mode(void) const override {
bool debug_mode(void) const override {
return debug_mode_;
}
virtual std::string debug_string(void) const override {
std::string debug_string(void) const override {
char buf[4096];
size_t size = snprintf(buf, sizeof(buf), "scan_cnt=%zu", get_scan_num());
return std::string(buf, size);
}
virtual int update(const ailego::Params &params) override;
int update(const ailego::Params &params) override;
int init(ContextType type);
+41 -44
View File
@@ -26,89 +26,86 @@ class VamanaStreamer : public IndexStreamer {
using ContextPointer = IndexStreamer::Context::Pointer;
VamanaStreamer(void);
virtual ~VamanaStreamer(void);
~VamanaStreamer(void) override;
VamanaStreamer(const VamanaStreamer &) = delete;
VamanaStreamer &operator=(const VamanaStreamer &) = delete;
protected:
virtual int init(const IndexMeta &imeta,
const ailego::Params &params) override;
int init(const IndexMeta &imeta, const ailego::Params &params) override;
virtual int cleanup(void) override;
int cleanup(void) override;
virtual Context::Pointer create_context(void) const override;
Context::Pointer create_context(void) const override;
virtual IndexProvider::Pointer create_provider(void) const override;
IndexProvider::Pointer create_provider(void) const override;
virtual int add_impl(uint64_t pkey, const void *query,
int add_impl(uint64_t pkey, const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
int add_with_id_impl(uint32_t id, const void *query,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
virtual int add_with_id_impl(uint32_t id, const void *query,
int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count, Context::Pointer &context) const override;
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta,
Context::Pointer &context) override;
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
Context::Pointer &context) const override;
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
uint32_t count,
Context::Pointer &context) const override;
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
ContextPointer &context) const override {
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
}
virtual int search_bf_by_p_keys_impl(
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
int search_bf_by_p_keys_impl(const void *query,
const std::vector<std::vector<uint64_t>> &p_keys,
const IndexQueryMeta &qmeta, uint32_t count,
ContextPointer &context) const override;
virtual const void *get_vector(uint64_t key) const override {
const void *get_vector(uint64_t key) const override {
return entity_->get_vector_by_key(key);
}
virtual int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
int get_vector(const uint64_t key,
IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector_by_key(key, block);
}
virtual const void *get_vector_by_id(uint32_t id) const override {
const void *get_vector_by_id(uint32_t id) const override {
return entity_->get_vector(id);
}
virtual int get_vector_by_id(
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
int get_vector_by_id(const uint32_t id,
IndexStorage::MemoryBlock &block) const override {
return entity_->get_vector(id, block);
}
virtual int open(IndexStorage::Pointer stg) override;
int open(IndexStorage::Pointer stg) override;
virtual int close(void) override;
int close(void) override;
virtual int flush(uint64_t checkpoint) override;
int flush(uint64_t checkpoint) override;
virtual int dump(const IndexDumper::Pointer &dumper) override;
int dump(const IndexDumper::Pointer &dumper) override;
virtual const Stats &stats(void) const override {
const Stats &stats(void) const override {
return stats_;
}
virtual const IndexMeta &meta(void) const override {
const IndexMeta &meta(void) const override {
return meta_;
}
virtual void print_debug_info() override;
void print_debug_info() override;
private:
inline int check_params(const void *query,
@@ -44,28 +44,28 @@ enum class VamanaStorageMode { kMmap = 0, kBufferPool = 1, kContiguous = 2 };
class VamanaStreamerEntity : public VamanaEntity {
public:
// Virtual interface implementation
virtual int cleanup() override;
virtual const VamanaEntity::Pointer clone() const override;
virtual key_t get_key(node_id_t id) const override;
virtual const void *get_vector(node_id_t id) const override;
virtual int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
virtual int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
virtual int get_vector(
int cleanup() override;
const VamanaEntity::Pointer clone() const override;
key_t get_key(node_id_t id) const override;
const void *get_vector(node_id_t id) const override;
int get_vector(const node_id_t id,
IndexStorage::MemoryBlock &block) const override;
int get_vector(const node_id_t *ids, uint32_t count,
const void **vecs) const override;
int get_vector(
const node_id_t *ids, uint32_t count,
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
virtual const Neighbors get_neighbors(node_id_t id) const override;
const Neighbors get_neighbors(node_id_t id) const override;
virtual int add_vector(key_t key, const void *vec, node_id_t *id) override;
virtual int add_vector_with_id(node_id_t id, const void *vec) override;
virtual int update_neighbors(
int add_vector(key_t key, const void *vec, node_id_t *id) override;
int add_vector_with_id(node_id_t id, const void *vec) override;
int update_neighbors(
node_id_t id,
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
virtual void add_neighbor(node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
virtual int dump(const IndexDumper::Pointer &dumper) override;
virtual void update_entry_point(node_id_t ep) override;
void add_neighbor(node_id_t id, uint32_t size,
node_id_t neighbor_id) override;
int dump(const IndexDumper::Pointer &dumper) override;
void update_entry_point(node_id_t ep) override;
// --- Neighbor distance storage ---
int ensure_dist_storage() override;
@@ -90,13 +90,13 @@ class VamanaStreamerEntity : public VamanaEntity {
VamanaStreamerEntity(IndexStreamer::Stats &stats);
~VamanaStreamerEntity();
virtual const void *get_vector_by_key(key_t key) const override {
const void *get_vector_by_key(key_t key) const override {
auto id = get_id(key);
return id == kInvalidNodeId ? nullptr : get_vector(id);
}
virtual int get_vector_by_key(
const key_t key, IndexStorage::MemoryBlock &block) const override {
int get_vector_by_key(const key_t key,
IndexStorage::MemoryBlock &block) const override {
auto id = get_id(key);
if (id != kInvalidNodeId) {
return get_vector(id, block);